GCC Code Coverage Report

Directory: ./
File: plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/xcom_base.cc
Date: 2022-12-06 21:40:42
Exec Total Coverage
Lines: 2771 3495 79.3%
Branches: 1937 3808 50.9%
Line Branch Exec Source
1 /* Copyright (c) 2012, 2022, Oracle and/or its affiliates.
2
3 This program is free software; you can redistribute it and/or modify
4 it under the terms of the GNU General Public License, version 2.0,
5 as published by the Free Software Foundation.
6
7 This program is also distributed with certain software (including
8 but not limited to OpenSSL) that is licensed under separate terms,
9 as designated in a particular file or component or in included license
10 documentation. The authors of MySQL hereby grant you an additional
11 permission to link the program and your derivative works with the
12 separately licensed software that they have included with MySQL.
13
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License, version 2.0, for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
22
23 #include <assert.h>
24 #include <errno.h>
25 #ifndef __STDC_FORMAT_MACROS
26 #define __STDC_FORMAT_MACROS
27 #endif
28 #ifndef _WIN32
29 #include <inttypes.h>
30 #endif
31 #include <limits.h>
32 #include <signal.h>
33 #include <stdio.h>
34 #include <stdlib.h>
35 #include <string.h>
36 #include <sys/time.h>
37 #include <sys/types.h>
38 #ifdef _MSC_VER
39 #include <stdint.h>
40 #endif
41
42 #ifndef _WIN32
43 #include <poll.h>
44 #endif
45
46 /**
47 @file
48 xcom/xcom_base.c
49 The new version of xcom is a major rewrite to allow
50 transmission of multiple messages from several sources
51 simultaneously without collision. The interface to xcom is largely
52 intact, one notable change is that xcom will consider the message
53 delivered as soon as it has got a majority. Consequently, the VP
54 set will not necessarily show all nodes which will actually
55 receive the message.
56
57 OHKFIX Add wait for complete last known node set to mimic the old
58 semantics.
59
60
61 IMPORTANT: What xcom does and what it does not do:
62
63 xcom messages are received in the same order on all nodes.
64
65 xcom guarantees that if a message is delivered to one node, it will
66 eventually be seen on all other nodes as well.
67
68 xcom messages are available to a crashed node when it comes up
69 again if at least one node which knows the value of the message
70 has not crashed. The size of the message cache is configurable.
71
72 OHKFIX Add logging to disk to make messages durable across system
73 crash and to increase the number of messages which may be cached.
74
75 There is no guarantee whatsoever about the order of messages from
76 different nodes, not even the order of multiple messages from the
77 same node. It is up to the client to impose such an order by
78 waiting on a message before it sends the next.
79
80 xcom can notify the client that a message has timed out, and in
81 that case will try to cancel the message, but it cannot guarantee
82 that a message which has timed out will not be delivered.
83
84 xcom attaches a node set to each message as it is delivered to the
85 client. This node set reflects the current node set that xcom
86 believes is active, it does not mean that the message has been
87 delivered yet to all nodes in the set. Neither does it mean that
88 the message has not been delivered to the nodes not in the set.
89
90 A cache of Paxos state machines is central to the new design. The
91 purpose of the cache is both to store a window of messages, and to
92 decouple the different parts of xcom, like message proposal,
93 message delivery and execution, and recovery. The old cache was
94 limited to caching messages, and a single state machine ran the
95 combined VP and Paxos algorithm. This constrained xcom to deliver
96 only a single message at a time.
97
98 Each instance of the Paxos state machine implements the basic
99 Paxos protocol. Unlike the cache in the old system, it is not
100 cleared when a site is deleted. This removes some problems
101 related to message delivery during site deletion. The cache is a
102 classic fixed size LRU with a hash index.
103
104 Some extensions to the basic Paxos algorithm has been implemented:
105
106 A node has ownership to all synodes with its own node number. Only
107 a node with node number N can propose a value for synode {X N},
108 where X is the sequence number, and N is the node number. Other
109 nodes can only propose the special value no_op for synode {X N}.
110 The reason for this is to retain the leaderless Paxos algorithm,
111 but to avoid collisions between nodes which are competing for the
112 same synode number. With this scheme, each node has its own unique
113 number series during normal operation. The scheme has the
114 following implications:
115
116 1. If a node N has not already proposed a value for the synode {X N},
117 it may at any time send a LEARN message to the other nodes with
118 the reserved value no_op, without going through phase 1 and 2 of
119 Paxos. This is because the other nodes are constrained to propose
120 no_op for this synode, so the final outcome will always be no_op.
121 To avoid unnecessary message transmission, a node will try to
122 broadcast the no_op LEARN messages by piggybacking the information
123 on the messages of the basic Paxos protocol.
124
125 2. Other nodes which want to find the value of synode {X N} may do
126 so by trying to get the value no_op accepted by following the
127 basic Paxos algorithm. The result will be the actual value
128 proposed by node N if it has done so, otherwise no_op. This will
129 typically only be necessary when a node is down, and the other
130 nodes need to find the values from the missing node in order to be
131 able to continue execution.
132
133 Messages are delivered in order to the client, and the order is
134 determined by the sequence number and the node number, with the
135 sequence number as the most significant part.
136
137 The xcom network interface has been redesigned and is now
138 implemented directly on top of TCP, and has so far been completely
139 trouble free. We use poll() or select() to implement non-blocking
140 send and receive, but libev could equally well have been used.
141
142 Multicast is implemented on top of unicast as before, but the
143 implementation is prepared to use real multicast with relatively
144 minor changes.
145
146 The roles of proposer, acceptor/learner, and executor are now
147 directly mapped to unique task types which interact with the Paxos
148 state machines, whereas the previous implementation folded all the
149 roles into a single event driven state machine.
150
151 The following terminology will be used:
152
153 A node is an instance of the xcom thread. There is only one instance
154 of the xcom thread in the agent.
155 A client is the application which is using xcom to send messages.
156 A thread is a real OS thread.
157 A task is a logical process. It is implemented by coroutines and
158 an explicit stack.
159
160 The implementation of tasks and non-blocking socket operations is
161 isolated in task.h and task.c.
162
163 A node will open a tcp connection to each of the other nodes. This
164 connection is used for all communication initiated by the node,
165 and replies to messages will arrive on the connection on which it
166 was sent.
167
168 static int tcp_server(task_arg);
169
170 The tcp_server listens on the xcom port and starts an
171 acceptor_learner_task whenever a new connection is detected.
172
173 static int tcp_reaper_task(task_arg);
174
175 Closes tcp connection which have been unused for too long.
176
177 static int sender_task(task_arg);
178
179 The sender_task waits for tcp messages on its input queue and
180 sends it on the tcp socket. If the socket is closed for any
181 reason, the sender_task will reconnect the socket. There is one
182 sender_task for each socket. The sender task exists mainly to
183 simplify the logic in the other tasks, but it could have been
184 replaced with a coroutine which handles the connection logic after
185 having reserved the socket for its client task.
186
187 static int generator_task(task_arg);
188
189 The generator_task reads messages from the client queue and moves
190 them into the input queue of the proposer_task.
191
192 OHKFIX Use a tcp socket instead of the client queue. We can then
193 remove the generator_task and let the acceptor_learner_task do the
194 dispatching.
195
196 static int proposer_task(task_arg);
197
198 Assign a message number to an outgoing message and try to get it
199 accepted. There may be several proposer tasks on each node
200 working in parallel. If there are multiple proposer tasks, xcom can
201 not guarantee that the messages will be sent in the same order as
202 received from the client.
203
204 static int acceptor_learner_task(task_arg);
205
206 This is the server part of the xcom thread. There is one
207 acceptor_learner_task for each node in the system. The acceptor
208 learner_task reads messages from the socket, finds the correct
209 Paxos state machine, and dispatches to the correct message handler
210 with the state machine and message as arguments.
211
212 static int reply_handler_task(task_arg);
213
214 The reply_handler_task does the same job as the
215 acceptor_learner_task, but listens on the socket which the node
216 uses to send messages, so it will handle only replies on that
217 socket.
218
219 static int executor_task(task_arg);
220
221 The ececutor_task waits for a Paxos message to be accpeted. When
222 the message is accepted, it is delivered to the client,
223 unless it is a no-op. In either case, the executor_task steps to
224 the next message and repeats the wait. If it times out waiting for
225 a message, it will try to get a no-op accepted.
226
227 static int alive_task(task_arg);
228
229 Sends i-am-alive to other nodes if there has been no normal traffic
230 for a while. It also pings nodes which seem to be inactive.
231
232 static int detector_task(task_arg);
233
234 The detector_task periodically scans the set of connections from
235 other nodes and sees if there has been any activity. If there has
236 been no activity for some time, it will assume that the node is
237 dead, and send a view message to the client.
238
239
240 Reconfiguration:
241
242 The xcom reconfiguration process is essentially the one described in
243 "Reconfiguring a State Machine" by Lamport et al. as the R-alpha
244 algorithm.
245 We execute the reconfiguration command immediately, but the config is
246 only valid after a delay of alpha messages.
247 The parameter alpha is the same as
248 EVENT_HORIZON in this implementation. :/static.*too_far
249 All tcp messages from beyond the event horizon will be ignored.
250
251 */
252 #include "xcom/xcom_profile.h"
253
254 #ifndef XCOM_STANDALONE
255 #include "my_compiler.h"
256 #endif
257 #include "xcom/x_platform.h"
258
259 #ifndef _WIN32
260 #include <arpa/inet.h>
261 #include <net/if.h>
262 #include <sys/ioctl.h>
263 #include <sys/socket.h>
264 #ifndef __linux__
265 #include <sys/sockio.h>
266 #endif
267 #endif
268
269 #if defined(_WIN32)
270 #include <windows.h>
271 #endif
272
273 #include <memory>
274
275 #include "xcom/app_data.h"
276 #include "xcom/get_synode_app_data.h"
277 #include "xcom/node_no.h"
278 #include "xcom/server_struct.h"
279 #include "xcom/simset.h"
280 #include "xcom/site_struct.h"
281 #include "xcom/task.h"
282 #include "xcom/task_net.h"
283 #include "xcom/task_os.h"
284 #include "xcom/xcom_base.h"
285 #include "xcom/xcom_common.h"
286 #include "xcom/xcom_detector.h"
287 #include "xcom/xcom_transport.h"
288 #include "xcom/xdr_utils.h"
289 #include "xdr_gen/xcom_vp.h"
290
291 #include "xcom/bitset.h"
292 #include "xcom/leader_info_data.h"
293 #include "xcom/node_list.h"
294 #include "xcom/node_set.h"
295 #include "xcom/pax_msg.h"
296 #include "xcom/site_def.h"
297 #include "xcom/sock_probe.h"
298 #include "xcom/synode_no.h"
299 #include "xcom/task_debug.h"
300 #include "xcom/task_net.h"
301 #include "xcom/xcom_cache.h"
302 #include "xcom/xcom_cfg.h"
303 #include "xcom/xcom_interface.h"
304 #include "xcom/xcom_memory.h"
305 #include "xcom/xcom_msg_queue.h"
306 #include "xcom/xcom_recover.h"
307 #include "xcom/xcom_statistics.h"
308 #include "xcom/xcom_vp_str.h"
309
310 #include "xcom/network/xcom_network_provider.h"
311
312 #ifndef XCOM_WITHOUT_OPENSSL
313 #ifdef _WIN32
314 /* In OpenSSL before 1.1.0, we need this first. */
315 #include <winsock2.h>
316 #endif /* _WIN32 */
317
318 #include <openssl/ssl.h>
319
320 #endif
321
322 #include <queue>
323 #include <tuple>
324
325 /* Defines and constants */
326
327 #define SYS_STRERROR_SIZE 512
328
329 /* Avoid printing the warning of protocol version mismatch too often */
330 #define PROTOVERSION_WARNING_TIMEOUT 600.0 /** Every 10 minutes */
331 static double protoversion_warning_time =
332 0.0; /** Timestamp of previous protoversion warning */
333
334 /* Skip prepare for first ballot */
335 #ifdef ALWAYS_THREEPHASE
336 int const threephase = 1;
337 #else
338 int const threephase = 0;
339 #endif
340
341 #include "xcom/retry.h"
342
343 #ifdef NODE_0_IS_ARBITRATOR
344 int ARBITRATOR_HACK = 1;
345 #else
346 int ARBITRATOR_HACK = 0;
347 #endif
348
349 static int const no_duplicate_payload = 1;
350
351 /* Use buffered read when reading messages from the network */
352 static int use_buffered_read = 1;
353
354 /* Used to handle OOM errors */
355 int oom_abort = 0;
356
357 /* Forward declarations */
358 long xcom_unique_long(void);
359 static int64_t socket_write(
360 connection_descriptor *wfd, void *_buf, uint32_t n,
361 connnection_write_method write_function = con_write);
362
363 static double wakeup_delay(double old);
364 static void note_snapshot(node_no node);
365
366 /* Task types */
367 static int proposer_task(task_arg arg);
368 static int executor_task(task_arg arg);
369 static int sweeper_task(task_arg arg);
370 extern int alive_task(task_arg arg);
371 extern int cache_manager_task(task_arg arg);
372 extern int detector_task(task_arg arg);
373
374 static int finished(pax_machine *p);
375 static int accepted(pax_machine *p);
376 static int started(pax_machine *p);
377 static synode_no first_free_synode_local(synode_no msgno);
378 static void free_forced_config_site_def();
379 static void activate_sweeper();
380 static void force_pax_machine(pax_machine *p, int enforcer);
381 static void propose_noop_2p(synode_no find, pax_machine *p);
382 static void handle_need_snapshot(linkage *reply_queue, pax_msg *pm);
383 static void handle_skip(site_def const *site, pax_machine *p, pax_msg *m);
384 static void paxos_fsm(pax_machine *paxos, site_def const *site,
385 paxos_event event, pax_msg *mess);
386 static inline bool is_leader(site_def *site);
387 int is_active_leader(node_no x, site_def *site);
388 static msg_handler *primary_dispatch_table();
389 static msg_handler *secondary_dispatch_table();
390 static void recompute_node_sets(site_def const *old_site, site_def *new_site);
391 void recompute_timestamps(detector_state const old_timestamp,
392 node_list const *old_nodes,
393 detector_state new_timestamp,
394 node_list const *new_nodes);
395 void analyze_leaders(site_def *site);
396 static inline int ignore_message(synode_no x, site_def *site, char const *dbg);
397
398 /* Global variables */
399
400 int xcom_shutdown = 0; /* Xcom_Shutdown flag */
401 synode_no executed_msg; /* The message we are waiting to execute */
402 synode_no max_synode; /* Max message number seen so far */
403 task_env *boot = nullptr;
404 task_env *detector = nullptr;
405 task_env *killer = nullptr;
406 task_env *net_boot = nullptr;
407 task_env *net_recover = nullptr;
408 void *xcom_thread_input = nullptr;
409
410 long xcom_debug_mask =
411 /* D_DETECT | */ D_FSM /* | D_FILEOP | D_CONS | D_BASE */ | D_TRANSPORT;
412 long xcom_dbg_stack[DBG_STACK_SIZE];
413 int xcom_dbg_stack_top = 0;
414
415 static void init_proposers();
416 void initialize_lsn(uint64_t n);
417
418 1207 void init_base_vars() {
419 1207 xcom_shutdown = 0; /* Xcom_Shutdown flag */
420 1207 executed_msg = null_synode; /* The message we are waiting to execute */
421 1207 max_synode = null_synode; /* Max message number seen so far */
422 1207 boot = nullptr;
423 1207 detector = nullptr;
424 1207 killer = nullptr;
425 1207 net_boot = nullptr;
426 1207 net_recover = nullptr;
427 1207 xcom_thread_input = nullptr;
428 1207 }
429
430 static task_env *executor = nullptr;
431 static task_env *sweeper = nullptr;
432 static task_env *retry = nullptr;
433 static task_env *proposer[PROPOSERS];
434 static task_env *alive_t = nullptr;
435 static task_env *cache_task = nullptr;
436
437 static uint32_t my_id = 0; /* Unique id of this instance */
438 872539 uint32_t get_my_xcom_id() { return my_id; }
439 static synode_no current_message; /* Current message number */
440 static synode_no
441 last_config_modification_id; /*Last configuration change proposal*/
442 static uint64_t lsn = 0; /* Current log sequence number */
443
444 86003 synode_no get_current_message() { return current_message; }
445
446 static channel prop_input_queue; /* Proposer task input queue */
447
448 enum class synode_allocation_type { todo = 0, local, remote, global };
449 enum class synode_reservation_status : int {
450 number_ok,
451 no_nodes,
452 delivery_timeout
453 };
454
455 #if 0
456 static char const *synode_allocation_type_to_str(synode_allocation_type x) {
457 switch (x) {
458 case synode_allocation_type::todo:
459 return "todo";
460 case synode_allocation_type::local:
461 return "local";
462 case synode_allocation_type::remote:
463 return "remote";
464 case synode_allocation_type::global:
465 return "global";
466 default:
467 return "";
468 }
469 }
470 #endif
471
472 // A pool of synode numbers implemented as a queue
473 struct synode_pool {
474 std::queue<std::pair<synode_no, synode_allocation_type>> data;
475 linkage queue;
476
477 1605 synode_pool() { link_init(&queue, TYPE_HASH("task_env")); }
478
479 456 void put(synode_no synode, synode_allocation_type allocation) {
480
1/2
✓ Branch 0 taken 456 times.
✗ Branch 1 not taken.
 456 data.push({synode, allocation});
481 456 task_wakeup(&queue);
482 456 }
483
484 456 auto get() {
485 456 auto retval = data.front();
486 456 data.pop();
487 456 return retval;
488 }
489
490 96918 bool empty() { return data.empty(); }
491 };
492
493 synode_pool synode_number_pool;
494
495 extern int client_boot_done;
496 extern int netboot_ok;
497
498 static linkage exec_wait = {
499 0, &exec_wait, &exec_wait}; /* Executor will wake up tasks sleeping here */
500
501 linkage detector_wait = {0, &detector_wait,
502 &detector_wait}; /* Detector sleeps here */
503
504 static struct {
505 int n;
506 unsigned long id[MAX_DEAD];
507 } dead_sites;
508
509 2812132 synode_no get_max_synode() { return max_synode; }
510
511 1837312 static bool_t is_latest_config(site_def const *const config) {
512 1837312 site_def const *const latest_config = get_site_def();
513
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1837312 times.
 1837312 assert(latest_config != nullptr);
514 1837312 return config == latest_config;
515 }
516
517 /**
518 * Get the first pending configuration that reconfigures the event horizon.
519 *
520 * Retrieve the first pending site_def, i.e. with the smallest start synod that
521 * is greater than executed_msg, that reconfigures the event horizon.
522 */
523 1831243 static site_def const *first_event_horizon_reconfig() {
524 1831243 site_def const *active_config = find_site_def(executed_msg);
525 1831243 xcom_event_horizon active_event_horizon = active_config->event_horizon;
526 1831243 site_def const *first_event_horizon_reconfig = nullptr;
527 1831243 site_def const *next_config = nullptr;
528 1831243 for (next_config = find_next_site_def(active_config->start);
529
4/4
✓ Branch 0 taken 479966 times.
✓ Branch 1 taken 1830731 times.
✓ Branch 2 taken 479454 times.
✓ Branch 3 taken 512 times.
 2310697 next_config != nullptr && first_event_horizon_reconfig == nullptr;
530 479454 next_config = find_next_site_def(next_config->start)) {
531
2/2
✓ Branch 0 taken 2610 times.
✓ Branch 1 taken 476844 times.
 479454 if (active_event_horizon != next_config->event_horizon) {
532 2610 first_event_horizon_reconfig = next_config;
533 }
534 }
535 1831243 return first_event_horizon_reconfig;
536 }
537
538 /**
539 * Get the latest pending configuration that reconfigures the event horizon.
540 *
541 * Retrieve the last pending site_def, i.e. with the greatest start synod that
542 * is greater than executed_msg, that reconfigures the event horizon.
543 */
544 6069 static site_def const *latest_event_horizon_reconfig() {
545 6069 site_def const *active_config = find_site_def(executed_msg);
546 6069 xcom_event_horizon previous_event_horizon = active_config->event_horizon;
547 6069 site_def const *last_event_horizon_reconfig = nullptr;
548 6069 site_def const *next_config = nullptr;
549 6069 for (next_config = find_next_site_def(active_config->start);
550
2/2
✓ Branch 0 taken 169 times.
✓ Branch 1 taken 6069 times.
 6238 next_config != nullptr;
551 169 next_config = find_next_site_def(next_config->start)) {
552
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 166 times.
 169 if (previous_event_horizon != next_config->event_horizon) {
553 3 previous_event_horizon = next_config->event_horizon;
554 3 last_event_horizon_reconfig = next_config;
555 }
556 }
557 6069 return last_event_horizon_reconfig;
558 }
559
560 /**
561 * Add the event horizon to the given base synod s.
562 *
563 * We are assuming right now that this function is used solely in the context of
564 * "we have received a reconfiguration command at synod s, when should it be
565 * scheduled to take effect?"
566 * The result of this function is *when* it should take effect.
567 *
568 * Common case: there are no configurations pending, or if there are, none of
569 * them reconfigure the event horizon. The common case result is:
570 *
571 * s + event_horizon(active_config) + 1
572 *
573 *
574 * If an event horizon reconfiguration R is pending, it means that the command C
575 * proposed for synod s is concurrent with R, i.e., s falls in the interval
576 * ]proposed(R), start(R)[.
577 *
578 * In this situation we apply the command C proposed for synod s *after* taking
579 * into account R's event horizon.
580 *
581 * This means that the result is:
582 *
583 * start(R) + event_horizon(R) + 1
584 */
585 #ifdef PERMISSIVE_EH_ACTIVE_CONFIG
586 /* purecov: begin deadcode */
587 static synode_no add_default_event_horizon(synode_no s) {
588 s.msgno += EVENT_HORIZON_MIN + 1;
589 return s;
590 }
591 /* purecov: end */
592 #endif
593
594 6069 static synode_no add_event_horizon(synode_no s) {
595 6069 site_def const *active_config = find_site_def(executed_msg);
596
1/2
✓ Branch 0 taken 6069 times.
✗ Branch 1 not taken.
 6069 if (active_config) {
597 6069 site_def const *pending_config = latest_event_horizon_reconfig();
598 6069 bool_t const no_event_horizon_reconfig_pending =
599 6069 (pending_config == nullptr);
600
6/6
✓ Branch 0 taken 133 times.
✓ Branch 1 taken 5936 times.
✓ Branch 2 taken 130 times.
✓ Branch 3 taken 3 times.
✓ Branch 4 taken 6066 times.
✓ Branch 5 taken 3 times.
 6069 if (is_latest_config(active_config) || no_event_horizon_reconfig_pending) {
601 6066 s.msgno = s.msgno + active_config->event_horizon + 1;
602 } else {
603 3 s.msgno = pending_config->start.msgno + pending_config->event_horizon + 1;
604 }
605 6069 return s;
606 } else { /* This is initial boot or recovery, we have no config */
607 #ifdef PERMISSIVE_EH_ACTIVE_CONFIG
608 return add_default_event_horizon(s);
609 #else
610 /* We should always have an active config */
611 /* purecov: begin deadcode */
612 ✗ assert(active_config != nullptr);
613 ✗ return null_synode;
614 /* purecov: end */
615 #endif
616 }
617 }
618
619 /**
620 Set node group
621 */
622 12329 void set_group(uint32_t id) {
623 IFDBG(D_NONE, FN; STRLIT("changing group id of global variables ");
624 NDBG((unsigned long)id, lu););
625 /* set_group_id(id); */
626 12329 current_message.group_id = id;
627 12329 executed_msg.group_id = id;
628 12329 max_synode.group_id = id;
629 12329 }
630
631 2348 static void bury_site(uint32_t id) {
632
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2348 times.
 2348 if (id != 0) {
633 ✗ dead_sites.id[dead_sites.n % MAX_DEAD] = id;
634 ✗ dead_sites.n = (dead_sites.n + 1) % MAX_DEAD;
635 }
636 2348 }
637
638 659780 static bool_t is_dead_site(uint32_t id) {
639 659780 int i = 0;
640
1/2
✓ Branch 0 taken 659780 times.
✗ Branch 1 not taken.
 659780 for (i = 0; i < MAX_DEAD; i++) {
641
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 659777 times.
 659780 if (dead_sites.id[i] == id)
642 3 return TRUE;
643
1/2
✓ Branch 0 taken 659777 times.
✗ Branch 1 not taken.
 659777 else if (dead_sites.id[i] == 0)
644 659777 return FALSE;
645 }
646 ✗ return FALSE;
647 }
648
649 extern node_set *init_node_set(node_set *set, u_int n);
650 extern node_set *alloc_node_set(node_set *set, u_int n);
651
652 #if 0
653 /* Find our previous message number. */
654 static synode_no decr_msgno(synode_no msgno)
655 {
656 synode_no ret = msgno;
657 ret.msgno--;
658 ret.node = get_nodeno(find_site_def(ret)); /* In case site and node number has changed */
659 return ret;
660 }
661 #endif
662
663 /* Find our next message number. */
664 364518 static synode_no incr_msgno(synode_no msgno) {
665 364518 synode_no ret = msgno;
666 364518 ret.msgno++;
667 364518 ret.node = get_nodeno(
668 find_site_def(ret)); /* In case site and node number has changed */
669 364518 return ret;
670 }
671
672 2067210 synode_no incr_synode(synode_no synode) {
673 2067210 synode_no ret = synode;
674 2067210 ret.node++;
675
2/2
✓ Branch 0 taken 1038422 times.
✓ Branch 1 taken 1028788 times.
 2067210 if (ret.node >= get_maxnodes(find_site_def(synode))) {
676 1038422 ret.node = 0;
677 1038422 ret.msgno++;
678 }
679 /* IFDBG(D_NONE, FN; SYCEXP(synode); SYCEXP(ret)); */
680 2067210 return ret; /* Change this if we change message number type */
681 }
682
683 #if 0
684 synode_no decr_synode(synode_no synode) {
685 synode_no ret = synode;
686 if (ret.node == 0) {
687 ret.msgno--;
688 ret.node = get_maxnodes(find_site_def(ret));
689 }
690 ret.node--;
691 return ret; /* Change this if we change message number type */
692 }
693 #endif
694
695 255176 static void skip_value(pax_msg *p) {
696 IFDBG(D_NONE, FN; SYCEXP(p->synode));
697 255176 p->op = learn_op;
698 255176 p->msg_type = no_op;
699 255176 }
700
701 /* Utilities and debug */
702
703 #ifndef _WIN32
704 /* Ignore this signal */
705 2362 static int ignoresig(int signum) {
706 struct sigaction act;
707 struct sigaction oldact;
708
709 2362 memset(&act, 0, sizeof(act));
710 2362 act.sa_handler = SIG_IGN;
711 2362 memset(&oldact, 0, sizeof(oldact));
712
713 2362 return sigaction(signum, &act, &oldact);
714 }
715 #else
716 #define SIGPIPE 0
717 static int ignoresig(int signum) { return 0; }
718 #endif
719
720 687753 static int recently_active(pax_machine *p) {
721 IFDBG(D_NONE, FN; SYCEXP(p->synode); STRLIT(" op "); PTREXP(p);
722 STRLIT(p->learner.msg ? pax_op_to_str(p->learner.msg->op) : "NULL");
723 NDBG(p->last_modified, f); NDBG(task_now(), f));
724
2/2
✓ Branch 0 taken 245123 times.
✓ Branch 1 taken 442630 times.
 932876 return p->last_modified != 0.0 &&
725
2/2
✓ Branch 0 taken 229390 times.
✓ Branch 1 taken 15733 times.
 932876 (p->last_modified + BUILD_TIMEOUT + median_time()) > task_now();
726 }
727
728 3566621 static inline int finished(pax_machine *p) {
729 IFDBG(D_NONE, FN; SYCEXP(p->synode); STRLIT(" op "); PTREXP(p);
730 STRLIT(p->learner.msg ? pax_op_to_str(p->learner.msg->op) : "NULL"););
731
4/4
✓ Branch 0 taken 1036615 times.
✓ Branch 1 taken 2530006 times.
✓ Branch 2 taken 39356 times.
✓ Branch 3 taken 997259 times.
 3605977 return p->learner.msg && (p->learner.msg->op == learn_op ||
732
1/2
✓ Branch 0 taken 39356 times.
✗ Branch 1 not taken.
 3605977 p->learner.msg->op == tiny_learn_op);
733 }
734
735 54687 int pm_finished(pax_machine *p) { return finished(p); }
736
737 208639 static inline int accepted(pax_machine *p) {
738 IFDBG(D_NONE, FN; SYCEXP(p->synode); STRLIT(" op "); PTREXP(p);
739 STRLIT(p->acceptor.msg ? pax_op_to_str(p->acceptor.msg->op) : "NULL"););
740
3/4
✓ Branch 0 taken 17871 times.
✓ Branch 1 taken 190768 times.
✓ Branch 2 taken 17871 times.
✗ Branch 3 not taken.
 208639 return p->acceptor.msg && p->acceptor.msg->op != initial_op;
741 }
742
743 15691 static inline int accepted_noop(pax_machine *p) {
744 IFDBG(D_NONE, FN; SYCEXP(p->synode); STRLIT(" op "); PTREXP(p);
745 STRLIT(p->acceptor.msg ? pax_op_to_str(p->acceptor.msg->op) : "NULL"););
746
4/4
✓ Branch 0 taken 259 times.
✓ Branch 1 taken 15432 times.
✓ Branch 2 taken 203 times.
✓ Branch 3 taken 56 times.
 15691 return accepted(p) && p->acceptor.msg->msg_type == no_op;
747 }
748
749 19920 static inline int noop_match(pax_machine *p, pax_msg *pm) {
750
4/4
✓ Branch 0 taken 15691 times.
✓ Branch 1 taken 4229 times.
✓ Branch 2 taken 203 times.
✓ Branch 3 taken 15488 times.
 19920 return pm->msg_type == no_op && accepted_noop(p);
751 }
752
753 156638 static inline int started(pax_machine *p) {
754
2/2
✓ Branch 0 taken 156614 times.
✓ Branch 1 taken 24 times.
 156638 return p->op != initial_op || (p->acceptor.promise.cnt > 0) ||
755
3/6
✗ Branch 0 not taken.
✓ Branch 1 taken 156614 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 142971 times.
✓ Branch 5 taken 13643 times.
 313228 (p->proposer.msg && (p->proposer.msg->op != initial_op)) ||
756
2/4
✓ Branch 0 taken 156638 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 142971 times.
 469890 accepted(p) || finished(p);
757 }
758
759 11418 void set_last_received_config(synode_no received_config_change) {
760 11418 last_config_modification_id = received_config_change;
761 11418 }
762
763 /* Definition of majority */
764 485715 static inline node_no max_check(site_def const *site) {
765 #ifdef MAXACCEPT
766 return MIN(get_maxnodes(site), MAXACCEPT);
767 #else
768 485715 return get_maxnodes(site);
769 #endif
770 }
771
772 static site_def *forced_config = nullptr;
773 67919 static int is_forcing_node(pax_machine const *p) { return p->enforcer; }
774 static int wait_forced_config = 0;
775
776 /* Definition of majority */
777 485715 static inline int majority(bit_set const *nodeset, site_def const *s, int all,
778 int delay [[maybe_unused]], int force) {
779 485715 node_no ok = 0;
780 485715 node_no i = 0;
781 485715 int retval = 0;
782 #ifdef WAIT_FOR_ALL_FIRST
783 double sec = task_now();
784 #endif
785 485715 node_no max = max_check(s);
786
787 /* IFDBG(D_NONE, FN; NDBG(max,lu); NDBG(all,d); NDBG(delay,d); NDBG(force,d));
788 */
789
790 /* Count nodes that has answered */
791
2/2
✓ Branch 0 taken 1122459 times.
✓ Branch 1 taken 485715 times.
 1608174 for (i = 0; i < max; i++) {
792
2/2
✓ Branch 0 taken 697852 times.
✓ Branch 1 taken 424607 times.
 1122459 if (BIT_ISSET(i, nodeset)) {
793 697852 ok++;
794 }
795 #ifdef WAIT_FOR_ALL_FIRST
796 else {
797 if (all) return 0; /* Delay until all nodes have answered */
798 if (delay && !may_be_dead(s->detected, i, sec)) {
799 return 0; /* Delay until all live nodes have answered */
800 }
801 }
802 #endif
803 }
804
805 /* If we are forcing messages, attempt to ensure consistency by
806 requiring all remaining nodes to agree. Forced_config points to
807 the config that should be used as acceptors in this
808 case. Another possibility is to use the original config and
809 count the number of live nodes, but since the force flag is
810 being used only to force a new config, it seems safer to use
811 the new config and no time-dependent info. Note that we are
812 counting the answers based on the normal config, but use the
813 number of nodes from forced_config. This is safe, since we can
814 assume that the nodes that are not in forced_config will never
815 answer. */
816
817
2/2
✓ Branch 0 taken 6533 times.
✓ Branch 1 taken 479182 times.
 485715 if (force) {
818 IFDBG(D_NONE, FN; STRLIT("force majority"); NDBG(ok, u); NDBG(max, u);
819 NDBG(get_maxnodes(forced_config), u));
820 6533 return ok == get_maxnodes(forced_config);
821 } else {
822 /* Have now seen answer from all live nodes */
823 #ifdef NODE_0_IS_ARBITRATOR
824 retval = all ? ok == max
825 : ok > max / 2 ||
826 (ARBITRATOR_HACK && (get_nodeno(s) == 0) && (2 == max));
827 #else
828
4/8
✗ Branch 0 not taken.
✓ Branch 1 taken 479182 times.
✓ Branch 2 taken 227691 times.
✓ Branch 3 taken 251491 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 227691 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 479182 retval = all ? ok == max : ok > max / 2 || (ARBITRATOR_HACK && (2 == max));
829 #endif
830 /* IFDBG(D_NONE, FN; NDBG(max,lu); NDBG(all,d); NDBG(delay,d);
831 * NDBG(retval,d)); */
832 479182 return retval;
833 }
834 }
835
836 #define IS_CONS_ALL(p) \
837 ((p)->proposer.msg->a ? (p)->proposer.msg->a->consensus == cons_all : 0)
838
839 /* See if a majority of acceptors have answered our prepare */
840 61554 static int prep_majority(site_def const *site, pax_machine const *p) {
841 61554 int ok = 0;
842
843
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 61554 times.
 61554 assert(p);
844
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 61554 times.
 61554 assert(p->proposer.prep_nodeset);
845
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 61554 times.
 61554 assert(p->proposer.msg);
846 /* IFDBG(D_NONE, FN; BALCEXP(p->proposer.bal)); */
847
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4418 times.
 61554 ok = majority(p->proposer.prep_nodeset, site, IS_CONS_ALL(p),
848
2/2
✓ Branch 0 taken 4418 times.
✓ Branch 1 taken 57136 times.
 61554 p->proposer.bal.cnt <= 1,
849
4/4
✓ Branch 0 taken 61419 times.
✓ Branch 1 taken 135 times.
✓ Branch 2 taken 3106 times.
✓ Branch 3 taken 58313 times.
 61554 p->proposer.msg->force_delivery || p->force_delivery);
850 61554 return ok;
851 }
852
853 /* See if a majority of acceptors have answered our propose */
854 424161 static int prop_majority(site_def const *site, pax_machine const *p) {
855 424161 int ok = 0;
856
857
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 424161 times.
 424161 assert(p);
858
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 424161 times.
 424161 assert(p->proposer.prop_nodeset);
859
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 424161 times.
 424161 assert(p->proposer.msg);
860 /* IFDBG(D_NONE, FN; BALCEXP(p->proposer.bal)); */
861
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 386191 times.
 424161 ok = majority(p->proposer.prop_nodeset, site, IS_CONS_ALL(p),
862
2/2
✓ Branch 0 taken 386191 times.
✓ Branch 1 taken 37970 times.
 424161 p->proposer.bal.cnt <= 1,
863
4/4
✓ Branch 0 taken 424001 times.
✓ Branch 1 taken 160 times.
✓ Branch 2 taken 3132 times.
✓ Branch 3 taken 420869 times.
 424161 p->proposer.msg->force_delivery || p->force_delivery);
864 424161 return ok;
865 }
866
867 /* Xcom thread */
868
869 static site_def *executor_site = nullptr;
870
871 1312 site_def const *get_executor_site() { return executor_site; }
872 86017 site_def *get_executor_site_rw() { return executor_site; }
873
874 static site_def *proposer_site = nullptr;
875
876 ✗ site_def const *get_proposer_site() { return proposer_site; }
877
878 /* delivered_msg may point to a no_op message, which will not actually be
879 * delivered */
880 static synode_no delivered_msg = NULL_SYNODE;
881
882 1757252 synode_no get_delivered_msg() { return delivered_msg; }
883
884 /* last_delivered_msg is the last synode we actually delivered */
885 static synode_no last_delivered_msg = NULL_SYNODE;
886 1313 synode_no get_last_delivered_msg() { return last_delivered_msg; }
887
888 3397 void init_xcom_base() {
889 IFDBG(D_NONE, FN);
890 3397 xcom_shutdown = 0;
891 3397 current_message = null_synode;
892 3397 executed_msg = null_synode;
893 3397 delivered_msg = null_synode;
894 3397 last_delivered_msg = null_synode;
895 3397 max_synode = null_synode;
896 3397 client_boot_done = 0;
897 3397 netboot_ok = 0;
898
899 3397 xcom_recover_init();
900 3397 my_id = new_id();
901 3397 push_site_def(nullptr);
902 /* update_servers(NULL); */
903 3397 xcom_cache_var_init();
904 3397 median_filter_init();
905 3397 link_init(&exec_wait, TYPE_HASH("task_env"));
906 3397 link_init(&detector_wait, TYPE_HASH("task_env"));
907 3397 link_init(&connect_wait, TYPE_HASH("task_env"));
908 3397 executor_site = nullptr;
909 3397 proposer_site = nullptr;
910
911 /** Reset lsn */
912 3397 initialize_lsn(0);
913 IFDBG(D_NONE, FN);
914 3397 }
915
916 3555 static void init_tasks() {
917 IFDBG(D_NONE, FN);
918 3555 set_task(&boot, nullptr);
919 3555 set_task(&net_boot, nullptr);
920 3555 set_task(&net_recover, nullptr);
921 3555 set_task(&killer, nullptr);
922 3555 set_task(&executor, nullptr);
923 3555 set_task(&retry, nullptr);
924 3555 set_task(&detector, nullptr);
925 3555 init_proposers();
926 3555 set_task(&alive_t, nullptr);
927 3555 set_task(&sweeper, nullptr);
928 3555 set_task(&cache_task, nullptr);
929 IFDBG(D_NONE, FN);
930 3555 }
931
932 /* Initialize the xcom thread */
933 1207 void xcom_thread_init() {
934 #ifndef NO_SIGPIPE
935 1207 signal(SIGPIPE, SIG_IGN);
936 #endif
937 1207 init_base_vars();
938 1207 init_site_vars();
939 1207 init_crc32c();
940 1207 xcom_srand48((long int)task_now());
941
942 1207 init_xcom_base();
943 1207 init_tasks();
944
945 /* Initialize input queue */
946 1207 channel_init(&prop_input_queue, TYPE_HASH("msg_link"));
947 1207 init_link_list();
948 1207 task_sys_init();
949
950 1207 init_cache();
951 1207 }
952
953 /* Empty the proposer input queue */
954 7949 static void empty_prop_input_queue() {
955 7949 empty_msg_channel(&prop_input_queue);
956 IFDBG(D_NONE, FN; STRLIT("prop_input_queue empty"));
957 7949 }
958
959 7949 static void empty_synode_number_pool() {
960
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7949 times.
 7949 while (!synode_number_pool.data.empty()) {
961 ✗ synode_number_pool.data.pop();
962 }
963 7949 }
964
965 /* De-initialize the xcom thread */
966 2355 void xcom_thread_deinit() {
967 IFDBG(D_BUG, FN; STRLIT("Empty proposer input queue"));
968 2355 empty_prop_input_queue();
969 IFDBG(D_BUG, FN; STRLIT("Empty synode number pool"));
970 2355 empty_synode_number_pool();
971 IFDBG(D_BUG, FN; STRLIT("Empty link free list"));
972 2355 empty_link_free_list();
973 IFDBG(D_BUG, FN; STRLIT("De-initialize cache"));
974 2355 deinit_cache();
975 2355 garbage_collect_servers();
976 IFDBG(D_BUG, FN; STRLIT("De-initialize network cache"));
977 2355 deinit_network_cache();
978 IFDBG(D_BUG, FN; STRLIT("De-initialize xcom_interface"));
979 2355 deinit_xcom_interface();
980 2355 }
981
982 #define PROP_ITER \
983 int i; \
984 for (i = 0; i < PROPOSERS; i++)
985
986 5745 static void init_proposers() {
987
2/2
✓ Branch 0 taken 57450 times.
✓ Branch 1 taken 5745 times.
 63195 PROP_ITER { set_task(&proposer[i], nullptr); }
988 5745 }
989
990 2197 static void create_proposers() {
991
2/2
✓ Branch 0 taken 21970 times.
✓ Branch 1 taken 2197 times.
 24167 PROP_ITER {
992 21970 set_task(&proposer[i], task_new(proposer_task, int_arg(i), "proposer_task",
993 XCOM_THREAD_DEBUG));
994 }
995 2197 }
996
997 static synode_no *proposer_synodes[PROPOSERS];
998
999 43870 static void add_proposer_synode(int i, synode_no *syn_ptr) {
1000
2/4
✓ Branch 0 taken 43870 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 43870 times.
✗ Branch 3 not taken.
 43870 if (i >= 0 && i < PROPOSERS) {
1001 43870 proposer_synodes[i] = syn_ptr;
1002 }
1003 43870 }
1004
1005 21900 static void remove_proposer_synode(int i) { add_proposer_synode(i, nullptr); }
1006
1007 78925 static synode_no get_proposer_synode(int i) {
1008
3/6
✓ Branch 0 taken 78925 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 78925 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 78925 times.
✗ Branch 5 not taken.
 78925 if (i >= 0 && i < PROPOSERS && proposer_synodes[i]) {
1009 78925 return *proposer_synodes[i];
1010 } else {
1011 ✗ return null_synode;
1012 }
1013 }
1014
1015 6661 static synode_no min_proposer_synode() {
1016 synode_no s_min;
1017 int i;
1018
1/2
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
 6661 for (i = 0; i < PROPOSERS; i++) {
1019 6661 s_min = get_proposer_synode(i);
1020
1/2
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
 6661 if (!synode_eq(null_synode, s_min)) break; // Initial value
1021 }
1022
2/2
✓ Branch 0 taken 66610 times.
✓ Branch 1 taken 6661 times.
 73271 for (; i < PROPOSERS; i++) {
1023
2/2
✓ Branch 0 taken 5654 times.
✓ Branch 1 taken 60956 times.
 66610 if (synode_lt(get_proposer_synode(i), s_min))
1024 5654 s_min = get_proposer_synode(i);
1025 }
1026 6661 return s_min;
1027 }
1028
1029 2190 static void terminate_proposers() {
1030
2/2
✓ Branch 0 taken 21900 times.
✓ Branch 1 taken 2190 times.
 24090 PROP_ITER { task_terminate(proposer[i]); }
1031 2190 }
1032
1033 4595 static void free_forced_config_site_def() {
1034 4595 free_site_def(forced_config);
1035 4595 forced_config = nullptr;
1036 4595 }
1037
1038 #if TASK_DBUG_ON
1039 [[maybe_unused]] static void dbg_proposers();
1040 static void dbg_proposers() {
1041 GET_GOUT;
1042 if (!IS_XCOM_DEBUG_WITH(XCOM_DEBUG_TRACE)) return;
1043 NDBG(PROPOSERS, d);
1044 {
1045 PROP_ITER { PPUT(proposer[i]); }
1046 }
1047 PRINT_GOUT;
1048 FREE_GOUT;
1049 }
1050 #endif
1051
1052 2197 static void set_proposer_startpoint() {
1053 IFDBG(D_NONE, FN; STRLIT("changing current message"));
1054
2/2
✓ Branch 0 taken 1360 times.
✓ Branch 1 taken 837 times.
 2197 if (synode_gt(max_synode, get_current_message())) {
1055
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1360 times.
 1360 if (max_synode.msgno <= 1)
1056 ✗ set_current_message(first_free_synode_local(max_synode));
1057 else
1058 1360 set_current_message(incr_msgno(first_free_synode_local(max_synode)));
1059 }
1060
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2197 times.
 2197 if (synode_gt(executed_msg, get_current_message())) {
1061 ✗ set_current_message(first_free_synode_local(executed_msg));
1062 }
1063 2197 }
1064
1065 /* Task functions */
1066
1067 static xcom_state_change_cb xcom_run_cb = nullptr;
1068 static xcom_state_change_cb xcom_terminate_cb = nullptr;
1069 static xcom_state_change_cb xcom_comms_cb = nullptr;
1070 static xcom_state_change_cb xcom_exit_cb = nullptr;
1071 static xcom_state_change_cb xcom_expel_cb = nullptr;
1072 static xcom_input_try_pop_cb xcom_try_pop_from_input_cb = nullptr;
1073 static xcom_recovery_cb MY_ATTRIBUTE((unused)) recovery_begin_cb = nullptr;
1074 static xcom_recovery_cb MY_ATTRIBUTE((unused)) recovery_restart_cb = nullptr;
1075 static xcom_recovery_cb MY_ATTRIBUTE((unused)) recovery_init_cb = nullptr;
1076 static xcom_recovery_cb MY_ATTRIBUTE((unused)) recovery_end_cb = nullptr;
1077
1078 2302 void set_xcom_run_cb(xcom_state_change_cb x) { xcom_run_cb = x; }
1079 2302 void set_xcom_exit_cb(xcom_state_change_cb x) { xcom_exit_cb = x; }
1080 2302 void set_xcom_comms_cb(xcom_state_change_cb x) { xcom_comms_cb = x; }
1081 2302 void set_xcom_expel_cb(xcom_state_change_cb x) { xcom_expel_cb = x; }
1082
1083 2302 void set_xcom_input_try_pop_cb(xcom_input_try_pop_cb pop) {
1084 2302 xcom_try_pop_from_input_cb = pop;
1085 2302 }
1086
1087 #ifdef XCOM_STANDALONE
1088 /* purecov: begin deadcode */
1089 void set_xcom_terminate_cb(xcom_state_change_cb x) { xcom_terminate_cb = x; }
1090 /* purecov: end */
1091
1092 /* purecov: begin deadcode */
1093 void set_xcom_recovery_begin_cb(xcom_recovery_cb x) { recovery_begin_cb = x; }
1094 /* purecov: end */
1095
1096 /* purecov: begin deadcode */
1097 void set_xcom_recovery_restart_cb(xcom_recovery_cb x) {
1098 recovery_restart_cb = x;
1099 }
1100 /* purecov: end */
1101
1102 /* purecov: begin deadcode */
1103 void set_xcom_recovery_init_cb(xcom_recovery_cb x) { recovery_init_cb = x; }
1104 /* purecov: end */
1105
1106 /* purecov: begin deadcode */
1107 void set_xcom_recovery_end_cb(xcom_recovery_cb x) { recovery_end_cb = x; }
1108 /* purecov: end */
1109 #endif
1110
1111 /**
1112 * These fields are used to signal XCom's request queue. After a request
1113 * is added, one will write 1 byte to warn local_server_task that it has work to
1114 * do.
1115 *
1116 * We use two types of signalling connection:
1117 * - An anonymous pipe, when possible, in POSIX compatible systems
1118 * - A regular socket connection, in Windows
1119 *
1120 * input_signal_connection is the connection_descriptor returned when one opens
1121 * a local signalling connection. It will contain either:
1122 * - The write side of a connection, in case of using a pipe OR;
1123 * - A bidirectional connection, when using a regular socket connection;
1124 *
1125 * input_signal_connection_pipe is the connection_descriptor that holds the read
1126 * side of a pipe connection. It is only allocated when we are able to have
1127 * a pipe connection.
1128 */
1129
1130 static connection_descriptor *input_signal_connection{nullptr};
1131
1132 connection_descriptor *input_signal_connection_pipe{nullptr};
1133 int pipe_signal_connections[2] = {-1, -1};
1134
1135 #ifndef XCOM_WITHOUT_OPENSSL
1136 ✗ static bool_t xcom_input_signal_connection_shutdown_ssl_wait_for_peer() {
1137 ✗ int ssl_error_code = 0;
1138 do {
1139 char buf[1024];
1140 ✗ ssl_error_code = SSL_read(input_signal_connection->ssl_fd, buf, 1024);
1141 ✗ } while (ssl_error_code > 0);
1142
1143 {
1144 bool_t const successful =
1145 ✗ (SSL_get_error(input_signal_connection->ssl_fd, ssl_error_code) ==
1146 ✗ SSL_ERROR_ZERO_RETURN);
1147 ✗ return successful;
1148 }
1149 }
1150
1151 ✗ static bool_t xcom_input_signal_connection_shutdown_ssl() {
1152 ✗ bool_t successful = FALSE;
1153
1154 ✗ int ssl_error_code = SSL_shutdown(input_signal_connection->ssl_fd);
1155
1156 ✗ bool_t const need_to_wait_for_peer_shutdown = (ssl_error_code == 0);
1157 ✗ bool_t const something_went_wrong = (ssl_error_code < 0);
1158 ✗ if (need_to_wait_for_peer_shutdown) {
1159 ✗ successful = xcom_input_signal_connection_shutdown_ssl_wait_for_peer();
1160 ✗ if (!successful) goto end;
1161 ✗ } else if (something_went_wrong) {
1162 ✗ goto end;
1163 }
1164
1165 ✗ ssl_free_con(input_signal_connection);
1166 ✗ successful = TRUE;
1167
1168 ✗ end:
1169 ✗ return successful;
1170 }
1171 #endif
1172
1173 2355 bool_t xcom_input_new_signal_connection(char const *address, xcom_port port) {
1174 2355 bool_t const SUCCESSFUL = TRUE;
1175 2355 bool_t const UNSUCCESSFUL = FALSE;
1176
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2355 times.
 2355 assert(input_signal_connection == nullptr);
1177
1178
1/2
✓ Branch 0 taken 2355 times.
✗ Branch 1 not taken.
 2355 if (input_signal_connection_pipe != nullptr) {
1179 2355 input_signal_connection =
1180 2355 (connection_descriptor *)malloc(sizeof(connection_descriptor));
1181 2355 input_signal_connection->fd = pipe_signal_connections[1];
1182 #ifndef XCOM_WITHOUT_OPENSSL
1183 2355 input_signal_connection->ssl_fd = nullptr;
1184 #endif
1185 2355 set_connected(input_signal_connection, CON_FD);
1186
1187
2/4
✓ Branch 0 taken 2355 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2355 times.
✗ Branch 3 not taken.
 2355 G_INFO("Successfully connected to the local XCom via anonymous pipe");
1188
1189 2355 return SUCCESSFUL;
1190 } else {
1191 /* purecov: begin deadcode */
1192 /* Try to connect. */
1193 ✗ input_signal_connection = open_new_local_connection(address, port);
1194 ✗ if (input_signal_connection->fd == -1) {
1195 ✗ return UNSUCCESSFUL;
1196 }
1197
1198 /* Have the server handle the rest of this connection using a local_server
1199 task. */
1200 ✗ if (xcom_client_convert_into_local_server(input_signal_connection) == 1) {
1201 ✗ G_TRACE(
1202 "Converted the signalling connection handler into a local_server "
1203 "task on the client side.");
1204
1205 #ifndef XCOM_WITHOUT_OPENSSL
1206 /* No more SSL in this connection. */
1207 ✗ if (Network_provider_manager::getInstance().get_running_protocol() ==
1208 XCOM_PROTOCOL) {
1209 ✗ bool_t const using_ssl = (input_signal_connection->ssl_fd != nullptr);
1210 ✗ if (using_ssl) {
1211 ✗ bool_t successful = xcom_input_signal_connection_shutdown_ssl();
1212 ✗ if (!successful) {
1213 ✗ G_ERROR(
1214 "Error shutting down SSL on XCom's signalling connection on "
1215 "the "
1216 "client side.");
1217 ✗ xcom_input_free_signal_connection();
1218 ✗ return UNSUCCESSFUL;
1219 }
1220 }
1221 }
1222 #endif
1223 ✗ G_INFO("Successfully connected to the local XCom via socket connection");
1224 ✗ return SUCCESSFUL;
1225 } else {
1226 ✗ G_INFO(
1227 "Error converting the signalling connection handler into a "
1228 "local_server task on the client side. This will result on a failure "
1229 "to join this node to a configuration");
1230 ✗ xcom_input_free_signal_connection();
1231 ✗ return UNSUCCESSFUL;
1232 }
1233 /* purecov: end */
1234 }
1235 }
1236
1237 107479 bool_t xcom_input_signal() {
1238 107479 bool_t successful = FALSE;
1239
2/2
✓ Branch 0 taken 107462 times.
✓ Branch 1 taken 17 times.
 107479 if (input_signal_connection != nullptr) {
1240 107462 unsigned char tiny_buf[1] = {0};
1241 int64_t error_code;
1242 107462 connnection_write_method to_write_function =
1243
1/2
✓ Branch 0 taken 107462 times.
✗ Branch 1 not taken.
 107462 input_signal_connection_pipe != nullptr ? con_pipe_write : con_write;
1244
1245 error_code =
1246
1/2
✓ Branch 0 taken 107461 times.
✗ Branch 1 not taken.
 107462 socket_write(input_signal_connection, tiny_buf, 1, to_write_function);
1247
1248 107461 successful = (error_code == 1);
1249 }
1250 107478 return successful;
1251 }
1252
1253 7004 void xcom_input_free_signal_connection() {
1254
2/2
✓ Branch 0 taken 2348 times.
✓ Branch 1 taken 4656 times.
 7004 if (input_signal_connection != nullptr) {
1255
1/2
✓ Branch 0 taken 2348 times.
✗ Branch 1 not taken.
 2348 if (input_signal_connection_pipe != nullptr) {
1256 2348 close(input_signal_connection->fd);
1257 } else {
1258 /* purecov: begin deadcode */
1259 ✗ close_open_connection(input_signal_connection);
1260 /* purecov: end */
1261 }
1262
1263 2348 free(input_signal_connection);
1264 2348 input_signal_connection = nullptr;
1265 }
1266 7004 }
1267
1268 #ifndef XCOM_WITHOUT_OPENSSL
1269 ✗ static int local_server_shutdown_ssl(connection_descriptor *con, void *buf,
1270 int n, int *ret) {
1271 DECL_ENV
1272 int ssl_error_code;
1273 bool_t need_to_wait_for_peer_shutdown;
1274 bool_t something_went_wrong;
1275 int64_t nr_read;
1276 ✗ ENV_INIT
1277 ✗ END_ENV_INIT
1278 END_ENV;
1279 ✗ *ret = 0;
1280 ✗ TASK_BEGIN
1281 ✗ ep->ssl_error_code = SSL_shutdown(con->ssl_fd);
1282 ✗ ep->need_to_wait_for_peer_shutdown = (ep->ssl_error_code == 0);
1283 ✗ ep->something_went_wrong = (ep->ssl_error_code < 0);
1284 ✗ if (ep->need_to_wait_for_peer_shutdown) {
1285 do {
1286 ✗ TASK_CALL(task_read(con, buf, n, &ep->nr_read));
1287 ✗ } while (ep->nr_read > 0);
1288 ✗ ep->ssl_error_code =
1289 ✗ SSL_get_error(con->ssl_fd, static_cast<int>(ep->nr_read));
1290 ✗ ep->something_went_wrong = (ep->ssl_error_code != SSL_ERROR_ZERO_RETURN);
1291 }
1292 ✗ if (ep->something_went_wrong) TERMINATE;
1293 ✗ ssl_free_con(con);
1294 ✗ *ret = 1;
1295 ✗ FINALLY
1296 ✗ TASK_END;
1297 }
1298 #endif
1299
1300 110471 int local_server(task_arg arg) {
1301 DECL_ENV
1302 connection_descriptor rfd;
1303 int ssl_shutdown_ret;
1304 unsigned char buf[1024]; /* arbitrary size */
1305 int64_t nr_read;
1306 xcom_input_request_ptr request;
1307 xcom_input_request_ptr next_request;
1308 pax_msg *request_pax_msg;
1309 pax_msg *reply_payload;
1310 linkage internal_reply_queue;
1311 msg_link *internal_reply;
1312 bool signaling_connection_error;
1313 connnection_read_method signal_read;
1314 2355 ENV_INIT
1315 2355 rfd.fd = -1;
1316 2355 ssl_shutdown_ret = 0;
1317 2355 memset(buf, 0, 1024);
1318 2355 nr_read = 0;
1319 2355 request = nullptr;
1320 2355 link_init(&internal_reply_queue, TYPE_HASH("msg_link"));
1321 2355 next_request = nullptr;
1322 2355 request_pax_msg = nullptr;
1323 2355 reply_payload = nullptr;
1324 2355 internal_reply = nullptr;
1325 2355 signaling_connection_error = false;
1326 2355 END_ENV_INIT
1327 END_ENV;
1328
6/11
✓ Branch 0 taken 2355 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 108116 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 2355 times.
✓ Branch 7 taken 2355 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 7 times.
✓ Branch 10 taken 2348 times.
 110471 TASK_BEGIN
1329
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2348 times.
 2348 assert(xcom_try_pop_from_input_cb != nullptr);
1330 {
1331 2348 connection_descriptor *arg_rfd = (connection_descriptor *)get_void_arg(arg);
1332 2348 ep->rfd = *arg_rfd;
1333
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2348 times.
 2348 if (input_signal_connection_pipe == nullptr) free(arg_rfd);
1334 }
1335
1336 // We will check if we have a pipe open or if we use a classic signalling
1337 // connection.
1338 2348 ep->signal_read =
1339
1/2
✓ Branch 0 taken 2348 times.
✗ Branch 1 not taken.
 2348 input_signal_connection_pipe != nullptr ? con_pipe_read : con_read;
1340
1341 #ifndef XCOM_WITHOUT_OPENSSL
1342 /* No more SSL in this connection. */
1343 2348 if (Network_provider_manager::getInstance().get_running_protocol() ==
1344
3/4
✓ Branch 0 taken 2284 times.
✓ Branch 1 taken 64 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2348 times.
 4632 XCOM_PROTOCOL &&
1345
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2284 times.
 2284 ep->rfd.ssl_fd) {
1346 ✗ TASK_CALL(local_server_shutdown_ssl(&ep->rfd, ep->buf, 1024,
1347 &ep->ssl_shutdown_ret));
1348 ✗ if (ep->ssl_shutdown_ret != 1) {
1349 ✗ G_ERROR(
1350 "Error shutting down SSL on XCom's signalling connection on the "
1351 "server side.");
1352 ✗ TERMINATE;
1353 }
1354 }
1355 #endif
1356
1357
1/2
✓ Branch 0 taken 108402 times.
✗ Branch 1 not taken.
 108402 while (!xcom_shutdown) {
1358 /* Wait for signal that there is work to consume from the queue. */
1359
1/2
✓ Branch 0 taken 108402 times.
✗ Branch 1 not taken.
 108402 if (!ep->signaling_connection_error) {
1360
10/14
✓ Branch 0 taken 108395 times.
✓ Branch 1 taken 108123 times.
✓ Branch 2 taken 2341 times.
✓ Branch 3 taken 106054 times.
✓ Branch 4 taken 108123 times.
✓ Branch 5 taken 106054 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 108116 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 108116 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 108116 times.
✓ Branch 13 taken 106054 times.
 322572 TASK_CALL(
1361 task_read(&ep->rfd, ep->buf, 1024, &ep->nr_read, ep->signal_read));
1362
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 106054 times.
 106054 if (ep->nr_read == 0) {
1363 /* purecov: begin inspected */
1364 ✗ G_WARNING("local_server: client closed the signalling connection?");
1365 ✗ ep->signaling_connection_error = true;
1366 /* purecov: end */
1367
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 106054 times.
 106054 } else if (ep->nr_read < 0) {
1368 /* purecov: begin inspected */
1369 IFDBG(D_NONE, FN; NDBG64(ep->nr_read));
1370 ✗ G_WARNING(
1371 "local_server: error reading from the signalling connection?");
1372 ✗ ep->signaling_connection_error = true;
1373 /* purecov: end */
1374 }
1375 }
1376
1377 /**
1378 * If an error occurs or if the client connection for the local server is
1379 * forcefully shutdown, we continue processing the queue until the end
1380 * resorting to time-based waits.
1381 */
1382
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 106054 times.
 106054 if (ep->signaling_connection_error) {
1383 ✗ TASK_DELAY(0.1);
1384 }
1385
1386 /* Pop, dispatch, and reply. */
1387 106054 ep->request = xcom_try_pop_from_input_cb();
1388
2/2
✓ Branch 0 taken 106360 times.
✓ Branch 1 taken 106054 times.
 212414 while (ep->request != nullptr) {
1389 /* Take ownership of the tail of the list, otherwise we lose it when we
1390 free ep->request. */
1391 106360 ep->next_request = xcom_input_request_extract_next(ep->request);
1392 106360 unchecked_replace_pax_msg(&ep->request_pax_msg,
1393 pax_msg_new_0(null_synode));
1394
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 106360 times.
 106360 assert(ep->request_pax_msg->refcnt == 1);
1395 106360 ep->request_pax_msg->op = client_msg;
1396
1397 /* Take ownership of the request's app_data, otherwise the app_data is
1398 freed with ep->request. */
1399 106360 ep->request_pax_msg->a = xcom_input_request_extract_app_data(ep->request);
1400 106360 ep->request_pax_msg->to = VOID_NODE_NO;
1401 106360 ep->request_pax_msg->force_delivery =
1402 106360 (ep->request_pax_msg->a->body.c_t == force_config_type);
1403 106360 dispatch_op(nullptr, ep->request_pax_msg, &ep->internal_reply_queue);
1404
2/2
✓ Branch 0 taken 2529 times.
✓ Branch 1 taken 103831 times.
 106360 if (!link_empty(&ep->internal_reply_queue)) {
1405 2529 ep->internal_reply =
1406 2529 (msg_link *)(link_extract_first(&ep->internal_reply_queue));
1407
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2529 times.
 2529 assert(ep->internal_reply->p);
1408
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2529 times.
 2529 assert(ep->internal_reply->p->refcnt == 1);
1409 /* We are going to take ownership of the pax_msg which has the reply
1410 payload, so we bump its reference count so that it is not freed by
1411 msg_link_delete. */
1412 2529 ep->reply_payload = ep->internal_reply->p;
1413 2529 ep->reply_payload->refcnt++;
1414 2529 msg_link_delete(&ep->internal_reply);
1415 /* There should only have been one reply. */
1416
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2529 times.
 2529 assert(link_empty(&ep->internal_reply_queue));
1417 } else {
1418 103831 ep->reply_payload = nullptr;
1419 }
1420 /* Reply to the request. */
1421 106360 xcom_input_request_reply(ep->request, ep->reply_payload);
1422 106360 xcom_input_request_free(ep->request);
1423 106360 ep->request = ep->next_request;
1424 }
1425 }
1426 ✗ FINALLY
1427 IFDBG(D_BUG, FN; STRLIT(" shutdown "); NDBG(ep->rfd.fd, d);
1428 NDBG(task_now(), f));
1429 /* Close the signalling connection. */
1430
1/2
✓ Branch 0 taken 2348 times.
✗ Branch 1 not taken.
 2348 if (!ep->signaling_connection_error) {
1431
1/2
✓ Branch 0 taken 2348 times.
✗ Branch 1 not taken.
 2348 if (input_signal_connection_pipe != nullptr &&
1432
2/2
✓ Branch 0 taken 2341 times.
✓ Branch 1 taken 7 times.
 2348 ep->rfd.fd != -1) { // We add -1 here, because in rare cases, the task
1433 // might have not been activated. Thus, it might
1434 // not have a reference to the socket to close.
1435 2341 close(ep->rfd.fd);
1436 2341 remove_and_wakeup(ep->rfd.fd);
1437 } else {
1438 7 shutdown_connection(&ep->rfd);
1439 }
1440 }
1441
1442 2348 unchecked_replace_pax_msg(&ep->request_pax_msg, nullptr);
1443 IFDBG(D_NONE, FN; NDBG(xcom_shutdown, d));
1444
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2348 times.
 2348 TASK_END;
1445 }
1446
1447 2355 static bool_t local_server_is_setup() {
1448 2355 return xcom_try_pop_from_input_cb != nullptr;
1449 }
1450
1451 static void init_time_queue();
1452 static int paxos_timer_task(task_arg arg [[maybe_unused]]);
1453
1454 2362 int xcom_taskmain2(xcom_port listen_port) {
1455 2362 init_xcom_transport(listen_port);
1456
1457 IFDBG(D_BUG, FN; STRLIT("enter taskmain"));
1458 2362 ignoresig(SIGPIPE);
1459
1460 {
1461 2362 result tcp_fd = {0, 0};
1462
1463 /*
1464 Setup networking
1465 */
1466
1/2
✓ Branch 0 taken 2362 times.
✗ Branch 1 not taken.
 2362 auto &net_manager = Network_provider_manager::getInstance();
1467 bool error_starting_network_provider =
1468
1/2
✓ Branch 0 taken 2362 times.
✗ Branch 1 not taken.
 2362 net_manager.start_active_network_provider();
1469
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 2355 times.
 2362 if (error_starting_network_provider) {
1470 /* purecov: begin inspected */
1471
4/8
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 7 times.
✗ Branch 7 not taken.
 7 g_critical("Unable to start %s Network Provider",
1472 Communication_stack_to_string::to_string(
1473 net_manager.get_running_protocol()));
1474
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 if (xcom_comms_cb) {
1475
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 xcom_comms_cb(XCOM_COMMS_ERROR);
1476 }
1477
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 if (xcom_terminate_cb) {
1478 ✗ xcom_terminate_cb(0);
1479 }
1480 7 goto cleanup;
1481 /* purecov: end */
1482 }
1483
1484 // We will use POSIX pipes for local queue signaling if we are not using WIN32
1485 #if !defined(_WIN32)
1486
1/2
✓ Branch 0 taken 2355 times.
✗ Branch 1 not taken.
 2355 if (local_server_is_setup()) {
1487 /* Launch local_server task to handle this connection. */
1488 {
1489
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2355 times.
 2355 if (pipe(pipe_signal_connections) == -1) {
1490 /* purecov: begin inspected */
1491 ✗ g_critical("Unable to start local signaling mechanism");
1492 ✗ if (xcom_comms_cb) {
1493 ✗ xcom_comms_cb(XCOM_COMMS_ERROR);
1494 }
1495 ✗ if (xcom_terminate_cb) {
1496 ✗ xcom_terminate_cb(0);
1497 }
1498 ✗ goto cleanup;
1499 /* purecov: end */
1500 }
1501
1/2
✓ Branch 0 taken 2355 times.
✗ Branch 1 not taken.
 2355 unblock_fd(pipe_signal_connections[0]);
1502
1503 /*
1504 Create the read side of input_signal_connection_pipe and create the
1505 local_server.
1506
1507 If one would to use regular sockets, this code is not executed and
1508 the local_server is created in the dispatch_op function.
1509 */
1510 2355 input_signal_connection_pipe =
1511 2355 (connection_descriptor *)malloc(sizeof(connection_descriptor));
1512 2355 input_signal_connection_pipe->fd = pipe_signal_connections[0];
1513 #ifndef XCOM_WITHOUT_OPENSSL
1514 2355 input_signal_connection_pipe->ssl_fd = nullptr;
1515 #endif
1516 2355 set_connected(input_signal_connection_pipe, CON_FD);
1517
1/2
✓ Branch 0 taken 2355 times.
✗ Branch 1 not taken.
 2355 task_new(local_server, void_arg(input_signal_connection_pipe),
1518 "local_server", XCOM_THREAD_DEBUG);
1519 }
1520 }
1521 #endif
1522
1523
1/2
✓ Branch 0 taken 2355 times.
✗ Branch 1 not taken.
 2355 if (xcom_comms_cb) {
1524
1/2
✓ Branch 0 taken 2355 times.
✗ Branch 1 not taken.
 2355 xcom_comms_cb(XCOM_COMMS_OK);
1525 }
1526
1527 IFDBG(D_NONE, FN; STRLIT("Creating tasks"));
1528
1529
1/2
✓ Branch 0 taken 2355 times.
✗ Branch 1 not taken.
 2355 task_new(incoming_connection_task, int_arg(tcp_fd.val), "tcp_server",
1530 XCOM_THREAD_DEBUG);
1531
1/2
✓ Branch 0 taken 2355 times.
✗ Branch 1 not taken.
 2355 task_new(tcp_reaper_task, null_arg, "tcp_reaper_task", XCOM_THREAD_DEBUG);
1532 #if defined(_WIN32)
1533 task_new(tcp_reconnection_task, null_arg, "tcp_reconnection_task",
1534 XCOM_THREAD_DEBUG);
1535 #endif
1536
1537 2355 init_time_queue();
1538
1/2
✓ Branch 0 taken 2355 times.
✗ Branch 1 not taken.
 2355 task_new(paxos_timer_task, null_arg, "paxos_timer_task", XCOM_THREAD_DEBUG);
1539 IFDBG(D_BUG, FN; STRLIT("XCOM is listening on "); NPUT(listen_port, d));
1540 }
1541
1542 #ifdef XCOM_STANDALONE
1543 if (recovery_init_cb) recovery_init_cb();
1544
1545 if (recovery_begin_cb) recovery_begin_cb();
1546 #endif
1547
1548 2355 task_loop();
1549 2355 cleanup:
1550
1551 #ifdef TASK_EVENT_TRACE
1552 dump_task_events();
1553 #endif
1554 // STOP NETWORK PROVIDERS
1555 2355 Network_provider_manager::getInstance().stop_all_network_providers();
1556
1557 2355 xcom_thread_deinit();
1558
1559 IFDBG(D_BUG, FN; STRLIT(" exit "); NDBG(xcom_dbg_stack_top, d);
1560 NDBG((unsigned)xcom_debug_mask, x));
1561 2355 xcom_debug_mask = 0;
1562 2355 xcom_dbg_stack_top = 0;
1563
2/2
✓ Branch 0 taken 2348 times.
✓ Branch 1 taken 7 times.
 2355 if (input_signal_connection_pipe != nullptr) {
1564 2348 ::xcom_input_free_signal_connection();
1565
1566 2348 free(input_signal_connection_pipe);
1567 2348 input_signal_connection_pipe = nullptr;
1568
1569 2348 pipe_signal_connections[0] = -1;
1570 2348 pipe_signal_connections[1] = -1;
1571 }
1572
1573
1/2
✓ Branch 0 taken 2355 times.
✗ Branch 1 not taken.
 2355 if (xcom_exit_cb) {
1574 2355 xcom_exit_cb(0);
1575 }
1576
1577 2355 return 1;
1578 }
1579
1580 /* Paxos message construction and sending */
1581
1582 /* Initialize a message for sending */
1583 633427 static void prepare(pax_msg *p, pax_op op) {
1584 633427 p->op = op;
1585 633427 p->reply_to = p->proposal;
1586 633427 }
1587
1588 /* Initialize a prepare_msg */
1589 48228 void init_prepare_msg(pax_msg *p) { prepare(p, prepare_op); }
1590
1591 25372 static int prepare_msg(pax_msg *p) {
1592 25372 init_prepare_msg(p);
1593 /* p->msg_type = normal; */
1594 25372 return send_to_acceptors(p, "prepare_msg");
1595 }
1596
1597 /* Initialize a noop_msg */
1598 22853 pax_msg *create_noop(pax_msg *p) {
1599 22853 init_prepare_msg(p);
1600 22853 p->msg_type = no_op;
1601 22853 return p;
1602 }
1603
1604 /* Initialize a read_msg */
1605 425940 static pax_msg *create_read(site_def const *site, pax_msg *p) {
1606 425940 p->msg_type = normal;
1607 425940 p->proposal.node = get_nodeno(site);
1608 425940 prepare(p, read_op);
1609 425940 return p;
1610 }
1611
1612 159259 static int skip_msg(pax_msg *p) {
1613 159259 prepare(p, skip_op);
1614 IFDBG(D_NONE, FN; STRLIT("skipping message "); SYCEXP(p->synode));
1615 159259 p->msg_type = no_op;
1616 159259 return send_to_all(p, "skip_msg");
1617 }
1618
1619 130523 static void brand_app_data(pax_msg *p) {
1620 130523 app_data_ptr a = p->a;
1621
2/2
✓ Branch 0 taken 112219 times.
✓ Branch 1 taken 130523 times.
 242742 while (a) {
1622 112219 a->app_key = p->synode;
1623 112219 a->group_id = p->synode.group_id;
1624 IFDBG(D_NONE, FN; PTREXP(a); SYCEXP(p->synode); SYCEXP(a->app_key));
1625 112219 a = a->next;
1626 }
1627 130523 }
1628
1629 109902 static synode_no my_unique_id(synode_no synode) {
1630
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 109902 times.
 109902 assert(my_id != 0);
1631 109902 site_def const *site = find_site_def(synode);
1632 /* Random number derived from node number and timestamp which uniquely defines
1633 * this instance */
1634 109902 synode.group_id = my_id;
1635 109902 synode.node = get_nodeno(site);
1636 109902 return synode;
1637 }
1638
1639 109902 static void set_unique_id(pax_msg *msg, synode_no synode) {
1640 109902 app_data_ptr a = msg->a;
1641
2/2
✓ Branch 0 taken 111042 times.
✓ Branch 1 taken 109902 times.
 220944 while (a) {
1642 111042 a->unique_id = synode;
1643 111042 a = a->next;
1644 }
1645 109902 }
1646
1647 130523 void init_propose_msg(pax_msg *p) {
1648 130523 p->op = accept_op;
1649 130523 p->reply_to = p->proposal;
1650 130523 brand_app_data(p);
1651 /* set_unique_id(p, my_unique_id(synode)); */
1652 130523 }
1653
1654 120285 static int send_propose_msg(pax_msg *p) {
1655 120285 return send_to_acceptors(p, "propose_msg");
1656 }
1657
1658 109825 static int propose_msg(pax_msg *p) {
1659 109825 init_propose_msg(p);
1660 109825 return send_propose_msg(p);
1661 }
1662
1663 119859 static void set_learn_type(pax_msg *p) {
1664 119859 p->op = learn_op;
1665 119859 p->msg_type = p->a ? normal : no_op;
1666 119859 }
1667
1668 /* purecov: begin deadcode */
1669 ✗ static void init_learn_msg(pax_msg *p) {
1670 ✗ set_learn_type(p);
1671 ✗ p->reply_to = p->proposal;
1672 }
1673
1674 ✗ static int send_learn_msg(site_def const *site, pax_msg *p) {
1675 IFDBG(D_NONE, FN; dbg_bitset(p->receivers, get_maxnodes(site)););
1676 ✗ return send_to_all_site(site, p, "learn_msg");
1677 }
1678 /* purecov: end */
1679
1680 118274 static pax_msg *create_tiny_learn_msg(pax_machine *pm, pax_msg *p) {
1681 118274 pax_msg *tiny_learn_msg = clone_pax_msg_no_app(p);
1682
1683 118274 ref_msg(tiny_learn_msg);
1684 118274 tiny_learn_msg->msg_type = p->a ? normal : no_op;
1685 118274 tiny_learn_msg->op = tiny_learn_op;
1686 118274 tiny_learn_msg->reply_to = pm->proposer.bal;
1687
1688 118274 return tiny_learn_msg;
1689 }
1690
1691 118268 static int send_tiny_learn_msg(site_def const *site, pax_msg *p) {
1692 118268 int retval = send_to_all_site(site, p, "tiny_learn_msg");
1693 118268 unref_msg(&p);
1694 118268 return retval;
1695 }
1696
1697 /* Proposer task */
1698
1699 25375 void prepare_push_3p(site_def const *site, pax_machine *p, pax_msg *msg,
1700 synode_no msgno, pax_msg_type msg_type) {
1701 IFDBG(D_NONE, FN; SYCEXP(msgno); NDBG(p->proposer.bal.cnt, d);
1702 NDBG(p->acceptor.promise.cnt, d));
1703 25375 BIT_ZERO(p->proposer.prep_nodeset);
1704 25375 p->proposer.bal.node = get_nodeno(site);
1705 {
1706
2/2
✓ Branch 0 taken 1355 times.
✓ Branch 1 taken 24020 times.
 25375 int maxcnt = MAX(p->proposer.bal.cnt, p->acceptor.promise.cnt);
1707 25375 p->proposer.bal.cnt = ++maxcnt;
1708 }
1709 25375 msg->synode = msgno;
1710 25375 msg->proposal = p->proposer.bal;
1711 25375 msg->msg_type = msg_type;
1712 25375 msg->force_delivery = p->force_delivery;
1713 25375 }
1714
1715 109828 void prepare_push_2p(site_def const *site, pax_machine *p) {
1716
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 109828 times.
 109828 assert(p->proposer.msg);
1717
1718 109828 BIT_ZERO(p->proposer.prop_nodeset);
1719 IFDBG(D_NONE, FN; SYCEXP(p->synode));
1720 109828 p->proposer.bal.cnt = 0;
1721 109828 p->proposer.bal.node = get_nodeno(site);
1722 109828 p->proposer.msg->proposal = p->proposer.bal;
1723 109828 p->proposer.msg->synode = p->synode;
1724 109828 p->proposer.msg->force_delivery = p->force_delivery;
1725 109828 }
1726
1727 109825 static void push_msg_2p(site_def const *site, pax_machine *p) {
1728 109825 prepare_push_2p(site, p);
1729 109825 propose_msg(p->proposer.msg);
1730 109825 }
1731
1732 25372 static void push_msg_3p(site_def const *site, pax_machine *p, pax_msg *msg,
1733 synode_no msgno, pax_msg_type msg_type) {
1734
2/2
✓ Branch 0 taken 537 times.
✓ Branch 1 taken 24835 times.
 25372 if (wait_forced_config) {
1735 537 force_pax_machine(p, 1);
1736 }
1737
1738
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 25372 times.
 25372 assert(msgno.msgno != 0);
1739 25372 prepare_push_3p(site, p, msg, msgno, msg_type);
1740
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 25372 times.
 25372 assert(p->proposer.msg);
1741 25372 prepare_msg(msg);
1742 IFDBG(D_NONE, FN; BALCEXP(msg->proposal); SYCEXP(msgno); STRLIT(" op ");
1743 STRLIT(pax_op_to_str(msg->op)));
1744 25372 }
1745
1746 /* Brand client message with unique ID */
1747 109902 static void brand_client_msg(pax_msg *msg, synode_no msgno) {
1748
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 109902 times.
 109902 assert(!synode_eq(msgno, null_synode));
1749 109902 set_unique_id(msg, my_unique_id(msgno));
1750 109902 }
1751
1752 6348 void xcom_send(app_data_ptr a, pax_msg *msg) {
1753 IFDBG(D_NONE, FN; PTREXP(a); SYCEXP(a->app_key); SYCEXP(msg->synode));
1754 6348 msg->a = a;
1755 6348 msg->op = client_msg;
1756 {
1757 6348 msg_link *link = msg_link_new(msg, VOID_NODE_NO);
1758 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_pax_msg(msg)));
1759 6348 channel_put(&prop_input_queue, &link->l);
1760 }
1761 6348 }
1762
1763 #define FNVSTART 0x811c9dc5
1764
1765 /* Fowler-Noll-Vo type multiplicative hash */
1766 10191 static uint32_t fnv_hash(unsigned char *buf, size_t length, uint32_t sum) {
1767 10191 size_t i = 0;
1768
2/2
✓ Branch 0 taken 1379182 times.
✓ Branch 1 taken 10191 times.
 1389373 for (i = 0; i < length; i++) {
1769 1379182 sum = sum * (uint32_t)0x01000193 ^ (uint32_t)buf[i];
1770 }
1771 10191 return sum;
1772 }
1773
1774 /**
1775 Create a new (hopefully unique) ID. The basic idea is to create a hash from
1776 the host ID and a timestamp.
1777 */
1778 3397 uint32_t new_id() {
1779
1/2
✓ Branch 0 taken 3397 times.
✗ Branch 1 not taken.
 3397 long id = xcom_unique_long();
1780
1/2
✓ Branch 0 taken 3397 times.
✗ Branch 1 not taken.
 3397 double timestamp = task_now();
1781 3397 uint32_t retval = 0;
1782
5/6
✓ Branch 0 taken 3397 times.
✓ Branch 1 taken 3397 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3397 times.
✓ Branch 4 taken 3397 times.
✓ Branch 5 taken 3397 times.
 10191 while (retval == 0 ||
1783 3397 is_dead_site(retval)) { /* Avoid returning 0 or already used site id */
1784 3397 retval = fnv_hash((unsigned char *)&id, sizeof(id), 0);
1785 3397 retval = fnv_hash((unsigned char *)&timestamp, sizeof(timestamp), retval);
1786 }
1787 3397 return retval;
1788 }
1789
1790 7713 static synode_no getstart(app_data_ptr a) {
1791 7713 synode_no retval = null_synode;
1792 /* If a->group_id is null_id, we set the group id from app_key.group_id,
1793 * which is hopefully not null_id. If it is, we're out of luck. */
1794
2/4
✓ Branch 0 taken 7713 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 7713 times.
 7713 if (a && a->group_id == null_id) {
1795 /* purecov: begin deadcode */
1796 ✗ a->group_id = a->app_key.group_id; /* app_key may have valid group */
1797 /* purecov: end */
1798 }
1799
2/2
✓ Branch 0 taken 31 times.
✓ Branch 1 taken 7682 times.
 7713 G_DEBUG("pid %d getstart group_id %x", xpid(), a->group_id);
1800
2/4
✓ Branch 0 taken 7713 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 7713 times.
 7713 if (!a || a->group_id == null_id) {
1801 ✗ retval.group_id = new_id();
1802 } else {
1803 7713 a->app_key.group_id = a->group_id;
1804 7713 retval = a->app_key;
1805
4/4
✓ Branch 0 taken 6876 times.
✓ Branch 1 taken 837 times.
✓ Branch 2 taken 6012 times.
✓ Branch 3 taken 1701 times.
 14589 if (get_site_def() &&
1806
2/2
✓ Branch 0 taken 6012 times.
✓ Branch 1 taken 864 times.
 6876 retval.msgno > 1) { /* Special case for initial boot of site */
1807 /* Not valid until after event horizon has been passed */
1808 6012 retval = add_event_horizon(retval);
1809 }
1810 }
1811 7713 return retval;
1812 }
1813
1814 #ifdef PERMISSIVE_EH_ACTIVE_CONFIG
1815 /* purecov: begin deadcode */
1816 synode_no get_default_start(app_data_ptr a) {
1817 synode_no retval = null_synode;
1818 /* If a->group_id is null_id, we set the group id from app_key.group_id,
1819 * which is hopefully not null_id. If it is, we're out of luck. */
1820 if (a && a->group_id == null_id) {
1821 a->group_id = a->app_key.group_id; /* app_key may have valid group */
1822 }
1823 G_DEBUG("pid %d getstart group_id %x", xpid(), a ? a->group_id : 0);
1824 if (!a || a->group_id == null_id) {
1825 retval.group_id = new_id();
1826 } else {
1827 a->app_key.group_id = a->group_id;
1828 retval = a->app_key;
1829 if (retval.msgno > 1) { /* Special case for initial boot of site */
1830 /* Not valid until after event horizon has been passed */
1831 retval = add_default_event_horizon(retval);
1832 }
1833 }
1834 return retval;
1835 }
1836 /* purecov: end */
1837 #endif
1838
1839 #if TASK_DBUG_ON
1840 /* purecov: begin deadcode */
1841 static void dump_xcom_node_names(site_def const *site) {
1842 u_int i;
1843 constexpr const size_t bufsize = NSERVERS * 256;
1844 char buf[bufsize]; /* Big enough */
1845 char *p = buf;
1846 if (!site) {
1847 G_INFO("pid %d no site", xpid());
1848 return;
1849 }
1850 *p = 0;
1851 for (i = 0; i < site->nodes.node_list_len; i++) {
1852 p = strncat(p, site->nodes.node_list_val[i].address, bufsize - 1);
1853 p = strncat(p, " ", bufsize - 1);
1854 }
1855 buf[bufsize - 1] = 0;
1856 G_INFO("pid %d node names %s", xpid(), buf);
1857 }
1858 /* purecov: end */
1859 #endif
1860
1861 12329 void site_install_action(site_def *site, cargo_type operation) {
1862 IFDBG(D_NONE, FN; NDBG(get_nodeno(get_site_def()), u));
1863
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 12329 times.
 12329 assert(site->event_horizon);
1864
4/4
✓ Branch 0 taken 11477 times.
✓ Branch 1 taken 852 times.
✓ Branch 2 taken 7985 times.
✓ Branch 3 taken 4344 times.
 23806 if (group_mismatch(site->start, max_synode) ||
1865
2/2
✓ Branch 0 taken 7133 times.
✓ Branch 1 taken 4344 times.
 11477 synode_gt(site->start, max_synode))
1866 7985 set_max_synode(site->start);
1867 12329 site->nodeno = xcom_find_node_index(&site->nodes);
1868 12329 push_site_def(site);
1869 IFDBG(D_NONE, dump_xcom_node_names(site));
1870 IFDBG(D_BUG, FN; SYCEXP(site->start); SYCEXP(site->boot_key);
1871 NUMEXP(site->max_active_leaders));
1872 IFDBG(D_BUG, FN; COPY_AND_FREE_GOUT(dbg_site_def(site)));
1873 12329 set_group(get_group_id(site));
1874
2/2
✓ Branch 0 taken 11456 times.
✓ Branch 1 taken 873 times.
 12329 if (get_maxnodes(get_site_def())) {
1875 11456 update_servers(site, operation);
1876 }
1877 12329 site->install_time = task_now();
1878
3/6
✓ Branch 0 taken 12329 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 12329 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 12329 times.
✗ Branch 5 not taken.
 12329 G_INFO(
1879 "Sucessfully installed new site definition. Start synode for this "
1880 "configuration is " SY_FMT ", boot key synode is " SY_FMT
1881 ", configured event horizon=%" PRIu32 ", my node identifier is %u",
1882 SY_MEM(site->start), SY_MEM(site->boot_key), site->event_horizon,
1883 get_nodeno(site));
1884 IFDBG(D_NONE, FN; NDBG(get_nodeno(site), u));
1885 IFDBG(D_NONE, FN; SYCEXP(site->start); SYCEXP(site->boot_key);
1886 NDBG(site->install_time, f));
1887 IFDBG(D_NONE, FN; NDBG(get_nodeno(site), u));
1888 ADD_DBG(
1889 D_BASE, add_event(EVENT_DUMP_PAD, string_arg("nodeno"));
1890 add_event(EVENT_DUMP_PAD, uint_arg(get_nodeno(site)));
1891 add_event(EVENT_DUMP_PAD, string_arg("site->boot_key"));
1892 add_synode_event(site->boot_key);
1893 /* add_event(EVENT_DUMP_PAD, uint_arg(chksum_node_list(&site->nodes))); */
1894 );
1895 12329 }
1896
1897 85 static void active_leaders(site_def *site, leader_array *leaders) {
1898 u_int i;
1899 u_int n;
1900 /* Synthesize leaders by copying all node names of active leaders */
1901
2/2
✓ Branch 0 taken 219 times.
✓ Branch 1 taken 85 times.
 304 for (i = 0, n = 0; i < site->nodes.node_list_len; i++) {
1902
2/2
✓ Branch 0 taken 157 times.
✓ Branch 1 taken 62 times.
 219 if (is_active_leader(i, site)) n++;
1903 }
1904 85 leaders->leader_array_len = n;
1905
1/2
✓ Branch 0 taken 85 times.
✗ Branch 1 not taken.
 85 if (n) {
1906 85 leaders->leader_array_val = static_cast<leader *>(
1907 85 xcom_calloc((size_t)leaders->leader_array_len, sizeof(leader)));
1908
2/2
✓ Branch 0 taken 219 times.
✓ Branch 1 taken 85 times.
 304 for (i = 0, n = 0; i < site->nodes.node_list_len; i++) {
1909
2/2
✓ Branch 0 taken 157 times.
✓ Branch 1 taken 62 times.
 219 if (is_active_leader(i, site)) {
1910 157 leaders->leader_array_val[n++].address =
1911 157 strdup(site->nodes.node_list_val[i].address);
1912 }
1913 }
1914 } else {
1915 ✗ leaders->leader_array_val = nullptr;
1916 }
1917 85 }
1918
1919 3 extern "C" void synthesize_leaders(leader_array *leaders) {
1920 // Default value meaning 'not set by client '
1921 3 leaders->leader_array_len = 0;
1922 3 leaders->leader_array_val = nullptr;
1923 3 }
1924
1925 159 static bool leaders_set_by_client(site_def const *site) {
1926 159 return site->leaders.leader_array_len != 0;
1927 }
1928
1929 1731 static site_def *create_site_def_with_start(app_data_ptr a, synode_no start) {
1930 1731 site_def *site = new_site_def();
1931 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&a->body.app_u_u.nodes)););
1932 1731 init_site_def(a->body.app_u_u.nodes.node_list_len,
1933 a->body.app_u_u.nodes.node_list_val, site);
1934 1731 site->start = start;
1935 1731 site->boot_key = a->app_key;
1936
1937 // If SINGLE_WRITER_ONLY is defined, ALL configs will be single writer. Used for
1938 // running all tests in single writer mode
1939 #ifdef SINGLE_WRITER_ONLY
1940 site->max_active_leaders = 1; /*Single writer */
1941 #else
1942 1731 site->max_active_leaders = active_leaders_all; /* Set to all nodes*/
1943 #endif
1944
1945 1731 return site;
1946 }
1947
1948 static xcom_proto constexpr single_writer_support = x_1_9;
1949
1950 1704 static site_def *install_ng_with_start(app_data_ptr a, synode_no start) {
1951
1/2
✓ Branch 0 taken 1704 times.
✗ Branch 1 not taken.
 1704 if (a) {
1952 1704 site_def *site = create_site_def_with_start(a, start);
1953 1704 site_def const *old_site = get_site_def();
1954
1955 // The reason why we need to recompute node sets and time stamps, is that
1956 // node sets and time stamps are stored in the site_def indexed by node
1957 // number, but they really are related to a specific node, not a specific
1958 // node number. When the site_def changes, the node number of a node may
1959 // change, thus invalidating the mapping from node numbers to node sets and
1960 // timestamps. But given the old and new definition, it is possible to
1961 // remap.
1962
3/4
✓ Branch 0 taken 867 times.
✓ Branch 1 taken 837 times.
✓ Branch 2 taken 867 times.
✗ Branch 3 not taken.
 1704 if (old_site && old_site->x_proto >= single_writer_support) {
1963 867 recompute_node_sets(old_site, site);
1964 867 recompute_timestamps(old_site->detected, &old_site->nodes, site->detected,
1965 867 &site->nodes);
1966 }
1967 1704 site_install_action(site, a->body.c_t);
1968 1704 return site;
1969 }
1970 ✗ return nullptr;
1971 }
1972
1973 1704 site_def *install_node_group(app_data_ptr a) {
1974 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("a->app_key"));
1975 add_synode_event(a->app_key););
1976
1/2
✓ Branch 0 taken 1704 times.
✗ Branch 1 not taken.
 1704 if (a)
1977 1704 return install_ng_with_start(a, getstart(a));
1978 else
1979 ✗ return nullptr;
1980 }
1981
1982 247778 void set_max_synode(synode_no synode) {
1983 247778 max_synode = synode; /* Track max synode number */
1984 IFDBG(D_BASE, FN; STRLIT("new "); SYCEXP(max_synode));
1985 247778 activate_sweeper();
1986 247778 }
1987
1988 234180 static int is_busy(synode_no s) {
1989 234180 pax_machine *p = hash_get(s);
1990
2/2
✓ Branch 0 taken 77542 times.
✓ Branch 1 taken 156638 times.
 234180 if (!p) {
1991 77542 return 0;
1992 } else {
1993 156638 return started(p);
1994 }
1995 }
1996
1997 109702 bool_t match_my_msg(pax_msg *learned, pax_msg *mine) {
1998 IFDBG(D_NONE, FN; PTREXP(learned->a);
1999 if (learned->a) SYCEXP(learned->a->unique_id); PTREXP(mine->a);
2000 if (mine->a) SYCEXP(mine->a->unique_id););
2001
3/4
✓ Branch 0 taken 108386 times.
✓ Branch 1 taken 1316 times.
✓ Branch 2 taken 108386 times.
✗ Branch 3 not taken.
 109702 if (learned->a && mine->a) { /* Both have app data, see if data is mine */
2002 108386 return synode_eq(learned->a->unique_id, mine->a->unique_id);
2003
2/4
✓ Branch 0 taken 1316 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1316 times.
 1316 } else if (!(learned->a || mine->a)) { /* None have app data, anything goes */
2004 ✗ return TRUE;
2005 } else { /* Definitely mismatch */
2006 1316 return FALSE;
2007 }
2008 }
2009
2010 /*
2011 * Initialize the log sequence number (lsn).
2012 */
2013 5594 void initialize_lsn(uint64_t n) { lsn = n; }
2014
2015 /**
2016 * Assign the next log sequence number (lsn) for a message.
2017 *
2018 * Initial propose sets lsn to msgno of the max message number as safe starting
2019 * point, otherwise lsn shall be ever increasing. lsn ensures sender order known
2020 * on receiver side, as messages may arrive "out of order" due to
2021 * retransmission. We use max_synode instead of current_message to avoid any
2022 * conflict with lsn allocated by a previous instance of the node.
2023 */
2024 112104 static uint64_t assign_lsn() {
2025
2/2
✓ Branch 0 taken 2197 times.
✓ Branch 1 taken 109907 times.
 112104 if (lsn == 0) {
2026 2197 initialize_lsn(max_synode.msgno);
2027 }
2028 112104 lsn++;
2029 IFDBG(D_EXEC, NDBG64(lsn));
2030 112104 return lsn;
2031 }
2032
2033 #if TASK_DBUG_ON
2034 /* purecov: begin deadcode */
2035 static int check_lsn(app_data_ptr a) {
2036 while (a) {
2037 if (!a->lsn) return 0;
2038 a = a->next;
2039 }
2040 return 1;
2041 }
2042 /* purecov: end */
2043 #endif
2044
2045 static void propose_noop(synode_no find, pax_machine *p);
2046
2047 /**
2048 * Checks if the given synod s is outside the event horizon.
2049 *
2050 * Common case: there are no configurations pending, or if there are, none of
2051 * them reconfigure the event horizon. The common case threshold is:
2052 *
2053 * last_executed_synod + event_horizon(active_config)
2054 *
2055 *
2056 * If an event horizon reconfiguration R is pending, it is possible that it
2057 * reduces the event horizon. In that case, it is possible that the threshold
2058 * above falls outside the new event horizon.
2059 *
2060 * For example, consider last_executed_synod = 42 and
2061 * event_horizon(active_config) = 10.
2062 * At this point this member participates in synods up to 52.
2063 * Now consider an event horizon reconfiguration that takes effect at synod 45,
2064 * which modifies the event horizon to 2. This means that when
2065 * last_executed_synod = 45, event_horizon(active_config) = 2. At this point
2066 * this member should only be able to participate in synods up to 47. The member
2067 * may have previously started processing messages directed to synods between 47
2068 * and 52, but will now ignore messages directed to those same synods.
2069 *
2070 * We do not want to start processing messages that will eventually fall out
2071 * of the event horizon. More importantly, the threshold above may not be safe
2072 * due to the exit logic of executor_task.
2073 *
2074 * When a node removes itself from the group on configuration C starting at
2075 * synod start(C), the exit logic relies on knowing *when* a majority has
2076 * executed synod start(C) - 1, i.e. the last message of the last configuration
2077 * to contain the leaving node.
2078 *
2079 * With a constant event horizon, we know that when synod
2080 * start(C) + event_horizon is learnt, it is because a majority already executed
2081 * or is ready to execute (and thus learned) synod start(C). This implies that a
2082 * majority already executed start(C) - 1.
2083 *
2084 * With a dynamic event horizon, we cannot be sure that when synod
2085 * start(C) + event_horizon(C) is learnt, a majority already executed or is
2086 * ready to execute synod start(C).
2087 * This is because it is possible for a new, smaller, event horizon to take
2088 * effect between start(C) and start(C) + event_horizon(C).
2089 * If that happens, the threshold above allows nodes to participate in synods
2090 * which are possibly beyond start(C) + event_horizon(C), which can lead to the
2091 * value of synod start(C) + event_horizon(C) being learnt without a majority
2092 * already having executed or being ready to execute synod start(C).
2093 *
2094 * In order to maintain the assumption made by the executor_task's exit logic,
2095 * when an event horizon reconfiguration R is pending we set the threshold to
2096 * the minimum between:
2097 *
2098 * last_executed_synod + event_horizon(active_config)
2099 *
2100 * and:
2101 *
2102 * start(R) - 1 + event_horizon(R)
2103 */
2104 1828633 static uint64_t too_far_threshold(xcom_event_horizon active_event_horizon) {
2105 1828633 return executed_msg.msgno + active_event_horizon;
2106 }
2107
2108 2610 static uint64_t too_far_threshold_new_event_horizon_pending(
2109 site_def const *new_config) {
2110 2610 uint64_t last_executed = executed_msg.msgno;
2111 /* compute normal threshold */
2112 uint64_t possibly_unsafe_threshold;
2113 2610 site_def const *active_config = find_site_def(executed_msg);
2114 2610 xcom_event_horizon active_event_horizon = active_config->event_horizon;
2115 2610 possibly_unsafe_threshold = last_executed + active_event_horizon;
2116 /* compute threshold taking into account new event horizon */ {
2117 uint64_t maximum_safe_threshold;
2118 xcom_event_horizon new_event_horizon;
2119 2610 uint64_t start_new_event_horizon = new_config->start.msgno;
2120 2610 new_event_horizon = new_config->event_horizon;
2121 2610 maximum_safe_threshold = start_new_event_horizon - 1 + new_event_horizon;
2122 /* use the minimum of both for safety */
2123
2/2
✓ Branch 0 taken 1930 times.
✓ Branch 1 taken 680 times.
 2610 return MIN(possibly_unsafe_threshold, maximum_safe_threshold);
2124 }
2125 }
2126
2127 1831243 static inline int too_far(synode_no s) {
2128 1831243 uint64_t threshold = 0;
2129 1831243 site_def const *active_config = find_site_def(executed_msg);
2130
1/2
✓ Branch 0 taken 1831243 times.
✗ Branch 1 not taken.
 1831243 if (active_config != nullptr) {
2131 1831243 site_def const *pending_config = first_event_horizon_reconfig();
2132 1831243 bool_t const no_event_horizon_reconfig_pending =
2133 1831243 (pending_config == nullptr);
2134
6/6
✓ Branch 0 taken 471441 times.
✓ Branch 1 taken 1359802 times.
✓ Branch 2 taken 468831 times.
✓ Branch 3 taken 2610 times.
✓ Branch 4 taken 1828633 times.
✓ Branch 5 taken 2610 times.
 1831243 if (is_latest_config(active_config) || no_event_horizon_reconfig_pending) {
2135 1828633 threshold = too_far_threshold(active_config->event_horizon);
2136 } else {
2137 2610 threshold = too_far_threshold_new_event_horizon_pending(pending_config);
2138 }
2139 } else {
2140 /* we have no configs, resort to default */
2141 ✗ threshold = too_far_threshold(EVENT_HORIZON_MIN);
2142 }
2143 1831243 return s.msgno >= threshold;
2144 }
2145
2146 #define GOTO(x) \
2147 { \
2148 IFDBG(D_NONE, STRLIT("goto "); STRLIT(#x)); \
2149 goto x; \
2150 }
2151
2152 108767 static inline int is_view(cargo_type x) { return x == view_msg; }
2153
2154 326688 static inline int is_config(cargo_type x) {
2155
4/4
✓ Branch 0 taken 321954 times.
✓ Branch 1 taken 3060 times.
✓ Branch 2 taken 315774 times.
✓ Branch 3 taken 6180 times.
 325014 return x == unified_boot_type || x == add_node_type ||
2156
4/4
✓ Branch 0 taken 315746 times.
✓ Branch 1 taken 28 times.
✓ Branch 2 taken 315666 times.
✓ Branch 3 taken 80 times.
 315774 x == remove_node_type || x == set_event_horizon_type ||
2157
5/6
✓ Branch 0 taken 325014 times.
✓ Branch 1 taken 1674 times.
✓ Branch 2 taken 315666 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 147 times.
✓ Branch 5 taken 315519 times.
 651702 x == force_config_type || x == set_max_leaders ||
2158 326688 x == set_leaders_type;
2159 }
2160
2161 static int wait_for_cache(pax_machine **pm, synode_no synode, double timeout);
2162 static int prop_started = 0;
2163 static int prop_finished = 0;
2164
2165 /* Find a free slot locally.
2166 Note that we will happily increment past the event horizon.
2167 The caller is thus responsible for checking the validity of the
2168 returned value by calling too_far() and ignore_message(). */
2169 110264 static synode_no local_synode_allocator(synode_no synode) {
2170
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 110264 times.
 110264 assert(!synode_eq(synode, null_synode));
2171
2172 // Ensure node number of synode is ours, whilst also ensuring that the synode
2173 // is monotonically increasing
2174 110264 node_no const my_nodeno = get_nodeno(find_site_def(synode));
2175
2/2
✓ Branch 0 taken 95484 times.
✓ Branch 1 taken 14780 times.
 110264 if (my_nodeno >= synode.node) {
2176 95484 synode.node = my_nodeno;
2177 } else {
2178 14780 synode = incr_msgno(synode);
2179 }
2180
2181
2/2
✓ Branch 0 taken 13664 times.
✓ Branch 1 taken 110264 times.
 123928 while (is_busy(synode)) {
2182 13664 synode = incr_msgno(synode);
2183 }
2184
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 110264 times.
 110264 assert(!synode_eq(synode, null_synode));
2185 110264 return synode;
2186 }
2187
2188 /* Find a likely free slot globally.
2189 Note that we will happily increment past the event horizon.
2190 The caller is thus responsible for checking the validity of the
2191 returned value by calling too_far() and ignore_message().
2192 The test for ignore_message() here is only valid until the
2193 event horizon. */
2194 ✗ static synode_no global_synode_allocator(site_def *site, synode_no synode) {
2195 ✗ assert(!synode_eq(synode, null_synode));
2196
2197 ✗ while (ignore_message(synode, site, "global_synode_allocator")) {
2198 ✗ synode = incr_synode(synode);
2199 }
2200 ✗ assert(!synode_eq(synode, null_synode));
2201 ✗ return synode;
2202 }
2203
2204 /* Find a free slot on remote leader */
2205 48231 static node_no remote_synode_allocator(site_def *site, app_data const &a) {
2206 static node_no distributor = 0; // Distribute requests equally among leaders
2207 48231 node_no maxnodes = get_maxnodes(site);
2208 48231 distributor = distributor % maxnodes; // Rescale in case site has changed
2209 48231 node_no i = distributor;
2210 // Ensure that current_message is associated with site
2211
2/2
✓ Branch 0 taken 20 times.
✓ Branch 1 taken 48211 times.
 48231 if (synode_lt(current_message, site->start)) {
2212 20 current_message = site->start;
2213 }
2214 for (;;) {
2215
4/4
✓ Branch 0 taken 48745 times.
✓ Branch 1 taken 233 times.
✓ Branch 2 taken 47629 times.
✓ Branch 3 taken 1349 times.
 97723 if (is_active_leader(i, site) &&
2216
2/2
✓ Branch 0 taken 47629 times.
✓ Branch 1 taken 1116 times.
 48745 !may_be_dead(site->detected, i,
2217 task_now())) { // Found leader, send request
2218 pax_msg *p =
2219 47629 pax_msg_new(current_message, site); // Message number does not matter
2220 IFDBG(D_CONS, FN; STRLIT("sending request "); NUMEXP(i);
2221 SYCEXP(current_message));
2222 47629 p->op = synode_request;
2223 47629 send_server_msg(site, i, p);
2224 47629 distributor = (i + 1) % maxnodes;
2225 47629 return i;
2226 }
2227 1349 i = (i + 1) % maxnodes;
2228
2/2
✓ Branch 0 taken 602 times.
✓ Branch 1 taken 747 times.
 1349 if (i == distributor) {
2229 /* There are no leaders, see if we should become leader. Note the special
2230 case for `force_config_type`. If we are in a network partition
2231 situation that must be healed using `force_config_type`, the leader
2232 might not be available and we might not be `iamthegreatest`. If we are
2233 the one tasked with `force_config_type` the entire system is relying on
2234 us to get consensus on `force_config_type` to unblock the group.
2235 Therefore, we self-allocate a synod for `force_config_type` to ensure
2236 the system makes progress. */
2237
3/6
✓ Branch 0 taken 602 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 602 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 602 times.
 602 if (iamthegreatest(site) || a.body.c_t == force_config_type) {
2238 // Grab message number and answer to self
2239 ✗ synode_no synode = global_synode_allocator(site, current_message);
2240 ✗ if (!too_far(synode)) {
2241 // We will grab this number, advance current_message
2242 ✗ set_current_message(incr_synode(synode));
2243 IFDBG(D_CONS, FN; STRLIT("grab message "); SYCEXP(synode);
2244 SYCEXP(current_message));
2245 ✗ synode_number_pool.put(synode, synode_allocation_type::global);
2246 }
2247 }
2248 602 return get_nodeno(site);
2249 }
2250 747 }
2251 }
2252
2253 #ifdef DELIVERY_TIMEOUT
2254 static bool check_delivery_timeout(site_def *site, double start_propose,
2255 app_data *a) {
2256 bool retval =
2257 (start_propose + a->expiry_time) < task_now() && !enough_live_nodes(site);
2258 if (retval) {
2259 DBGOUT_ASSERT(check_lsn(a), STRLIT("NULL lsn"));
2260 IFDBG(D_NONE, FN; STRLIT("timeout -> delivery_failure"));
2261 deliver_to_app(NULL, a, delivery_failure);
2262 }
2263 return retval;
2264 }
2265 #endif
2266
2267 186478 static int reserve_synode_number(synode_allocation_type *synode_allocation,
2268 site_def **site, synode_no *msgno,
2269 int *remote_retry, app_data *a,
2270 synode_reservation_status *ret) {
2271 186478 *ret = synode_reservation_status::number_ok; // Optimistic, will be reset if
2272 // necessary
2273 DECL_ENV
2274 int dummy;
2275 113420 ENV_INIT
2276 113420 END_ENV_INIT
2277 END_ENV;
2278
2279
6/11
✓ Branch 0 taken 113420 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 48231 times.
✓ Branch 3 taken 24827 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 113420 times.
✓ Branch 7 taken 113420 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 113420 times.
 186478 TASK_BEGIN
2280 do {
2281 113432 *synode_allocation = synode_allocation_type::todo;
2282 IFDBG(D_CONS, FN; SYCEXP(current_message));
2283 113432 *site = find_site_def_rw(current_message);
2284
2/2
✓ Branch 0 taken 109808 times.
✓ Branch 1 taken 3624 times.
 113432 if (is_leader(*site)) { // Use local synode allocator
2285 109808 *msgno = local_synode_allocator(current_message);
2286 IFDBG(D_CONS, FN; SYCEXP(outer_ep->msgno));
2287 109808 *synode_allocation = synode_allocation_type::local;
2288 } else { // Cannot use local, try remote
2289 // Get synode number from another leader
2290 3624 *remote_retry = 0;
2291
2/2
✓ Branch 0 taken 48231 times.
✓ Branch 1 taken 456 times.
 48687 while (synode_number_pool.empty()) {
2292 // get_maxnodes(get_site_def()) > 0 is a precondition for
2293 // `remote_synode_allocator`.
2294
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 48231 times.
 48231 if (get_maxnodes(get_site_def()) == 0) {
2295 ✗ TASK_DELAY(0.1);
2296 ✗ TASK_RETURN(synode_reservation_status::no_nodes);
2297 }
2298 #if TASK_DBUG_ON
2299 node_no allocator_node =
2300 #endif
2301 48231 remote_synode_allocator(get_site_def_rw(),
2302 *a); // Send request for synode, use
2303 // latest config
2304 48231 if (*remote_retry > 10) {
2305 IFDBG(D_BUG, FN; NUMEXP(outer_ep->self); NUMEXP(allocator_node);
2306 SYCEXP(executed_msg); SYCEXP(current_message);
2307 SYCEXP(outer_ep->msgno); SYCEXP(get_site_def_rw()->start));
2308 }
2309
1/2
✓ Branch 0 taken 48231 times.
✗ Branch 1 not taken.
 48231 if (synode_number_pool.empty()) { // Only wait if still empty
2310
4/6
✗ Branch 0 not taken.
✓ Branch 1 taken 48231 times.
✓ Branch 2 taken 48231 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 3168 times.
✓ Branch 5 taken 45063 times.
 96462 TIMED_TASK_WAIT(&synode_number_pool.queue,
2311 0.1); // Wait for incoming synode
2312 }
2313 45063 (*remote_retry)++;
2314 }
2315 456 std::tie(*msgno, *synode_allocation) = synode_number_pool.get();
2316 IFDBG(D_CONS, FN; SYCEXP(outer_ep->msgno));
2317 }
2318
2319 // Update site to match synode
2320 110264 *site = proposer_site = find_site_def_rw(*msgno);
2321
2322 // Set the global current message for all number allocators
2323 110264 set_current_message(incr_synode(*msgno));
2324
2325
2/2
✓ Branch 0 taken 24827 times.
✓ Branch 1 taken 110252 times.
 135079 while (too_far(*msgno)) { /* Too far ahead of executor */
2326
4/6
✗ Branch 0 not taken.
✓ Branch 1 taken 24827 times.
✓ Branch 2 taken 24827 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 12 times.
✓ Branch 5 taken 24815 times.
 49654 TIMED_TASK_WAIT(&exec_wait, 0.2);
2327 IFDBG(D_NONE, FN; SYCEXP(ep->msgno); TIMECEXP(ep->start_propose);
2328 TIMECEXP(outer_ep->client_msg->p->a->expiry_time);
2329 TIMECEXP(task_now()); NDBG(enough_live_nodes(outer_ep->site), d));
2330 #ifdef DELIVERY_TIMEOUT
2331 if (check_delivery_timeout(outer_ep->site, outer_ep->start_propose,
2332 outer_ep->client_msg->p->a)) {
2333 TASK_RETURN(delivery_timeout);
2334 }
2335 #endif
2336 }
2337 // Filter out busy or ignored message numbers
2338
6/6
✓ Branch 0 taken 110249 times.
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 9 times.
✓ Branch 3 taken 110240 times.
✓ Branch 4 taken 12 times.
✓ Branch 5 taken 110240 times.
 110252 } while (is_busy(*msgno) || ignore_message(*msgno, *site, "proposer_task"));
2339 110240 FINALLY
2340
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 113420 times.
 113420 TASK_END;
2341 }
2342
2343 /* Send messages by fetching from the input queue and trying to get it accepted
2344 by a Paxos instance */
2345 341106 static int proposer_task(task_arg arg) {
2346 DECL_ENV
2347 int self; /* ID of this proposer task */
2348 pax_machine *p; /* Pointer to Paxos instance */
2349 msg_link *client_msg; /* The client message we are trying to push */
2350 synode_no msgno;
2351 pax_msg *prepare_msg;
2352 double start_propose;
2353 double start_push;
2354 double delay;
2355 site_def *site;
2356 size_t size;
2357 size_t nr_batched_app_data;
2358 int remote_retry;
2359 synode_allocation_type synode_allocation;
2360 21970 ENV_INIT
2361 21970 END_ENV_INIT
2362 END_ENV;
2363
2364 341106 synode_reservation_status reservation_status{
2365 synode_reservation_status::number_ok};
2366
2367
9/17
✓ Branch 0 taken 21970 times.
✓ Branch 1 taken 129983 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 73058 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 116095 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 21970 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 21970 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 21970 times.
✓ Branch 13 taken 21970 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✓ Branch 16 taken 21970 times.
 341106 TASK_BEGIN
2368
2369 21970 ep->self = get_int_arg(arg);
2370 21970 ep->p = nullptr;
2371 21970 ep->client_msg = nullptr;
2372 21970 ep->prepare_msg = nullptr;
2373 21970 ep->start_propose = 0.0;
2374 21970 ep->start_push = 0.0;
2375 21970 ep->delay = 0.0;
2376 21970 ep->msgno = current_message;
2377 21970 ep->site = nullptr;
2378 21970 ep->size = 0;
2379 21970 ep->nr_batched_app_data = 0;
2380 21970 ep->remote_retry = 0;
2381 21970 ep->synode_allocation = synode_allocation_type::todo;
2382 21970 add_proposer_synode(ep->self, &ep->msgno);
2383 IFDBG(D_NONE, FN; NDBG(ep->self, d); NDBG(task_now(), f));
2384
2385
1/2
✓ Branch 0 taken 130694 times.
✗ Branch 1 not taken.
 130694 while (!xcom_shutdown) { /* Loop until no more work to do */
2386 /* Wait for client message */
2387
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 130694 times.
 130694 assert(!ep->client_msg);
2388
8/12
✓ Branch 0 taken 130053 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 129983 times.
✓ Branch 4 taken 129983 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 18520 times.
✓ Branch 7 taken 111463 times.
✓ Branch 8 taken 130053 times.
✓ Branch 9 taken 112104 times.
✓ Branch 10 taken 112104 times.
✗ Branch 11 not taken.
 260677 CHANNEL_GET(&prop_input_queue, &ep->client_msg, msg_link);
2389 112104 prop_started++;
2390 IFDBG(D_NONE, FN; PTREXP(ep->client_msg->p->a); STRLIT("extracted ");
2391 SYCEXP(ep->client_msg->p->a->app_key));
2392
2393 /* Grab rest of messages in queue as well, but never batch config messages,
2394 * which need a unique number */
2395
2396 /* The batch is limited either by size or number of batched app_datas.
2397 * We limit the number of elements because the XDR deserialization
2398 * implementation is recursive, and batching too many app_datas will cause a
2399 * call stack overflow. */
2400
6/6
✓ Branch 0 taken 107653 times.
✓ Branch 1 taken 4451 times.
✓ Branch 2 taken 101305 times.
✓ Branch 3 taken 6348 times.
✓ Branch 4 taken 101305 times.
✓ Branch 5 taken 10799 times.
 219757 if (!is_config(ep->client_msg->p->a->body.c_t) &&
2401 107653 !is_view(ep->client_msg->p->a->body.c_t)) {
2402
1/2
✓ Branch 0 taken 101305 times.
✗ Branch 1 not taken.
 101305 ep->size = app_data_size(ep->client_msg->p->a);
2403 101305 ep->nr_batched_app_data = 1;
2404 101305 while (AUTOBATCH && ep->size <= MAX_BATCH_SIZE &&
2405
6/8
✓ Branch 0 taken 102396 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 102396 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1141 times.
✓ Branch 5 taken 101255 times.
✓ Branch 6 taken 1141 times.
✓ Branch 7 taken 101255 times.
 204792 ep->nr_batched_app_data <= MAX_BATCH_APP_DATA &&
2406 102396 !link_empty(&prop_input_queue
2407 .data)) { /* Batch payloads into single message */
2408 msg_link *tmp;
2409 app_data_ptr atmp;
2410
2411
2/12
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 1141 times.
✓ Branch 10 taken 1141 times.
✗ Branch 11 not taken.
 1141 CHANNEL_GET(&prop_input_queue, &tmp, msg_link);
2412 1141 atmp = tmp->p->a;
2413
1/2
✓ Branch 0 taken 1141 times.
✗ Branch 1 not taken.
 1141 ep->size += app_data_size(atmp);
2414 1141 ep->nr_batched_app_data++;
2415 /* Abort batching if config or too big batch */
2416
2/2
✓ Branch 0 taken 1091 times.
✓ Branch 1 taken 23 times.
 2255 if (is_config(atmp->body.c_t) || is_view(atmp->body.c_t) ||
2417
5/6
✓ Branch 0 taken 1114 times.
✓ Branch 1 taken 27 times.
✓ Branch 2 taken 1091 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 50 times.
✓ Branch 5 taken 1091 times.
 3346 ep->nr_batched_app_data > MAX_BATCH_APP_DATA ||
2418
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1091 times.
 1091 ep->size > MAX_BATCH_SIZE) {
2419
1/2
✓ Branch 0 taken 50 times.
✗ Branch 1 not taken.
 50 channel_put_front(&prop_input_queue, &tmp->l);
2420 50 break;
2421 }
2422 ADD_T_EV(seconds(), __FILE__, __LINE__, "batching");
2423
2424 1091 tmp->p->a = nullptr; /* Steal this payload */
2425
1/2
✓ Branch 0 taken 1091 times.
✗ Branch 1 not taken.
 1091 msg_link_delete(&tmp); /* Get rid of the empty message */
2426 1091 atmp->next = ep->client_msg->p->a; /* Add to list of app_data */
2427 /* G_TRACE("Batching %s %s",
2428 * cargo_type_to_str(ep->client_msg->p->a->body.c_t), */
2429 /* cargo_type_to_str(atmp->body.c_t)); */
2430 1091 ep->client_msg->p->a = atmp;
2431 IFDBG(D_NONE, FN; PTREXP(ep->client_msg->p->a); STRLIT("extracted ");
2432 SYCEXP(ep->client_msg->p->a->app_key));
2433 }
2434 }
2435
2436
1/2
✓ Branch 0 taken 112104 times.
✗ Branch 1 not taken.
 112104 ep->start_propose = task_now();
2437 112104 ep->delay = 0.0;
2438
2439
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 112104 times.
 112104 assert(!ep->client_msg->p->a->chosen);
2440
2441 /* It is a new message */
2442
2443
2/4
✓ Branch 0 taken 112104 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 112104 times.
 112104 assert(!synode_eq(current_message, null_synode));
2444
2445 /* Assign a log sequence number only on initial propose */
2446 {
2447
1/2
✓ Branch 0 taken 112104 times.
✗ Branch 1 not taken.
 112104 uint64_t prop_lsn = assign_lsn();
2448 112104 app_data_ptr ap = ep->client_msg->p->a;
2449 /* Assign to all app_data structs */
2450
2/2
✓ Branch 0 taken 113234 times.
✓ Branch 1 taken 112104 times.
 225338 while (ap) {
2451 113234 ap->lsn = prop_lsn;
2452 113234 ap = ap->next;
2453 }
2454 }
2455 DBGOUT_ASSERT(check_lsn(ep->client_msg->p->a), STRLIT("NULL lsn"));
2456 112104 retry_new:
2457 /* Find a free slot */
2458
12/18
✓ Branch 0 taken 113420 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 186478 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 113420 times.
✓ Branch 5 taken 73058 times.
✓ Branch 6 taken 3180 times.
✓ Branch 7 taken 110240 times.
✓ Branch 8 taken 73058 times.
✓ Branch 9 taken 110240 times.
✗ Branch 10 not taken.
✓ Branch 11 taken 73058 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 73058 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✓ Branch 16 taken 73058 times.
✓ Branch 17 taken 110240 times.
 259536 TASK_CALL(reserve_synode_number(&ep->synode_allocation, &ep->site,
2459 &ep->msgno, &ep->remote_retry,
2460 ep->client_msg->p->a, &reservation_status));
2461
2462 // Check result of reservation
2463
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 110240 times.
 110240 if (reservation_status == synode_reservation_status::no_nodes) {
2464 ✗ GOTO(retry_new);
2465
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 110240 times.
 110240 } else if (reservation_status ==
2466 synode_reservation_status::delivery_timeout) {
2467 ✗ GOTO(next);
2468 }
2469 // If we get here, we have a valid synode number
2470
2/4
✓ Branch 0 taken 110240 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 110240 times.
 110240 assert(!synode_eq(ep->msgno, null_synode));
2471
2472 /* See if we can do anything with this message */
2473
6/8
✓ Branch 0 taken 110240 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 110240 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 338 times.
✓ Branch 5 taken 109902 times.
✓ Branch 6 taken 338 times.
✓ Branch 7 taken 109902 times.
 110240 if (!ep->site || get_nodeno(ep->site) == VOID_NODE_NO) {
2474 /* Give up */
2475 DBGOUT_ASSERT(check_lsn(ep->client_msg->p->a), STRLIT("NULL lsn"));
2476 IFDBG(D_NONE, FN; STRLIT("delivery_failure "); SYCEXP(ep->msgno);
2477 PTREXP(ep->site); NDBG(get_nodeno(ep->site), u));
2478
1/2
✓ Branch 0 taken 338 times.
✗ Branch 1 not taken.
 338 deliver_to_app(nullptr, ep->client_msg->p->a, delivery_failure);
2479 338 GOTO(next);
2480 }
2481
2482
1/2
✓ Branch 0 taken 109902 times.
✗ Branch 1 not taken.
 109902 brand_client_msg(ep->client_msg->p, ep->msgno);
2483
2484 for (;;) { /* Loop until the client message has been learned */
2485 /* Get a Paxos instance to send the client message */
2486
2487
6/18
✓ Branch 0 taken 109902 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 109902 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 109902 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 109902 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 109902 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 109902 times.
 109902 TASK_CALL(wait_for_cache(&ep->p, ep->msgno, 60));
2488
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 109902 times.
 109902 if (!ep->p) {
2489 ✗ G_MESSAGE("Could not get a pax_machine for msgno %lu. Retrying",
2490 (unsigned long)ep->msgno.msgno);
2491 ✗ goto retry_new;
2492 }
2493
2494
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 109902 times.
 109902 assert(ep->p);
2495
2/2
✓ Branch 0 taken 27 times.
✓ Branch 1 taken 109875 times.
 109902 if (ep->client_msg->p->force_delivery)
2496 27 ep->p->force_delivery = ep->client_msg->p->force_delivery;
2497 {
2498
1/2
✓ Branch 0 taken 109902 times.
✗ Branch 1 not taken.
 109902 [[maybe_unused]] int lock = lock_pax_machine(ep->p);
2499
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 109902 times.
 109902 assert(!lock);
2500 }
2501
2502 /* Set the client message as current proposal */
2503
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 109902 times.
 109902 assert(ep->client_msg->p);
2504
2/4
✓ Branch 0 taken 109902 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 109902 times.
✗ Branch 3 not taken.
 109902 replace_pax_msg(&ep->p->proposer.msg, clone_pax_msg(ep->client_msg->p));
2505
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 109902 times.
 109902 if (ep->p->proposer.msg == nullptr) {
2506 ✗ g_critical(
2507 "Node %u has run out of memory while sending a message and "
2508 "will now exit.",
2509 get_nodeno(proposer_site));
2510 ✗ terminate_and_exit(); /* Tell xcom to stop */
2511 ✗ TERMINATE;
2512 }
2513
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 109902 times.
 109902 assert(ep->p->proposer.msg);
2514 PAX_MSG_SANITY_CHECK(ep->p->proposer.msg);
2515
2516 /* Create the prepare message */
2517
1/2
✓ Branch 0 taken 109902 times.
✗ Branch 1 not taken.
 109902 unchecked_replace_pax_msg(&ep->prepare_msg,
2518
1/2
✓ Branch 0 taken 109902 times.
✗ Branch 1 not taken.
 109902 pax_msg_new(ep->msgno, ep->site));
2519 IFDBG(D_NONE, FN; PTREXP(ep->client_msg->p->a); STRLIT("pushing ");
2520 SYCEXP(ep->msgno));
2521 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_app_data(ep->prepare_msg->a)));
2522
2523 /* Use 3 phase algorithm if threephase is set or we are forcing or we have
2524 already accepted something, which may happen if another node has timed
2525 out waiting for this node and proposed a no_op, which we have accepted.
2526
2527 We *must* use 3 phase algorithm if the synode was allocated by
2528 ourselves using `global_synode_allocator`. This is last resort synode
2529 allocation that does not guarantee we will be the only Proposer using
2530 it. Therefore, for correctness we must use regular 3 phase Paxos,
2531 because we may have dueling Proposers.
2532 */
2533
3/4
✓ Branch 0 taken 109825 times.
✓ Branch 1 taken 77 times.
✓ Branch 2 taken 109825 times.
✗ Branch 3 not taken.
 109902 if (threephase || ep->p->force_delivery || ep->p->acceptor.promise.cnt ||
2534
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 109825 times.
 109825 ep->synode_allocation == synode_allocation_type::global) {
2535
1/2
✓ Branch 0 taken 77 times.
✗ Branch 1 not taken.
 77 push_msg_3p(ep->site, ep->p, ep->prepare_msg, ep->msgno, normal);
2536 } else {
2537
1/2
✓ Branch 0 taken 109825 times.
✗ Branch 1 not taken.
 109825 push_msg_2p(ep->site, ep->p);
2538 }
2539
2540
1/2
✓ Branch 0 taken 109902 times.
✗ Branch 1 not taken.
 109902 ep->start_push = task_now();
2541
2542
1/2
✓ Branch 0 taken 116095 times.
✗ Branch 1 not taken.
 116095 while (!finished(ep->p)) { /* Try to get a value accepted */
2543 /* We will wake up periodically, and whenever a message arrives */
2544
8/14
✓ Branch 0 taken 116095 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 116095 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 116095 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 116095 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 116095 times.
✓ Branch 10 taken 116095 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 200 times.
✓ Branch 13 taken 115895 times.
 232190 TIMED_TASK_WAIT(&ep->p->rv, ep->delay = wakeup_delay(ep->delay));
2545
3/6
✓ Branch 0 taken 115895 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 115895 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 115895 times.
 231790 if (!synode_eq(ep->msgno, ep->p->synode) ||
2546
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 115895 times.
 115895 ep->p->proposer.msg == nullptr) {
2547 IFDBG(D_NONE, FN; STRLIT("detected stolen state machine, retry"););
2548 /* unlock_pax_machine(ep->p); */
2549 ✗ GOTO(retry_new); /* Need to break out of both loops,
2550 and we have no "exit named
2551 loop" construction */
2552 }
2553
3/6
✓ Branch 0 taken 115895 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 115895 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 115895 times.
✗ Branch 5 not taken.
 115895 assert(synode_eq(ep->msgno, ep->p->synode) && ep->p->proposer.msg);
2554
2/2
✓ Branch 0 taken 109702 times.
✓ Branch 1 taken 6193 times.
 115895 if (finished(ep->p)) break;
2555 {
2556
1/2
✓ Branch 0 taken 6193 times.
✗ Branch 1 not taken.
 6193 double now = task_now();
2557 #ifdef DELIVERY_TIMEOUT
2558 if ((ep->start_propose + ep->client_msg->p->a->expiry_time) < now) {
2559 IFDBG(D_NONE, FN; STRLIT("timeout when pushing ");
2560 SYCEXP(ep->msgno); SYCEXP(executed_msg));
2561 /* Proposing a no-op here is a last ditch effort to cancel the
2562 failed message. If any of the currently reachable nodes have
2563 participated in the failed consensus round, it is equivalent to
2564 retrying a final time, otherwise we could get a no-op
2565 accepted. Proposing a no-op is always harmless.
2566 Having a timeout on delivery and telling the client is really
2567 contrary to the spirit of
2568 Paxos, since we cannot guarantee that the message has not been
2569 delivered, but at the moment, MCM depends on it.
2570 Proposing a no-op here increases the probability that the outcome
2571 matches what we tell MCM about the outcome. */
2572 propose_noop(ep->msgno, ep->p);
2573 DBGOUT_ASSERT(check_lsn(ep->client_msg->p->a), STRLIT("NULL lsn"));
2574 IFDBG(D_NONE, FN; STRLIT("timeout -> delivery_failure"));
2575 deliver_to_app(ep->p, ep->client_msg->p->a, delivery_failure);
2576 unlock_pax_machine(ep->p);
2577 GOTO(next);
2578 }
2579 #endif
2580
2/2
✓ Branch 0 taken 3521 times.
✓ Branch 1 taken 2672 times.
 6193 if ((ep->start_push + ep->delay) <= now) {
2581 PAX_MSG_SANITY_CHECK(ep->p->proposer.msg);
2582 IFDBG(D_NONE, FN; STRLIT("retry pushing "); SYCEXP(ep->msgno));
2583 IFDBG(D_NONE, FN;
2584 COPY_AND_FREE_GOUT(dbg_app_data(ep->prepare_msg->a)););
2585 IFDBG(D_NONE, BALCEXP(ep->p->proposer.bal);
2586 BALCEXP(ep->p->acceptor.promise));
2587
1/2
✓ Branch 0 taken 3521 times.
✗ Branch 1 not taken.
 3521 push_msg_3p(ep->site, ep->p, ep->prepare_msg, ep->msgno, normal);
2588 3521 ep->start_push = now;
2589 }
2590 }
2591 }
2592 /* When we get here, we know the value for this message number,
2593 but it may not be the value we tried to push,
2594 so loop until we have a successful push. */
2595
1/2
✓ Branch 0 taken 109702 times.
✗ Branch 1 not taken.
 109702 unlock_pax_machine(ep->p);
2596 IFDBG(D_NONE, FN; STRLIT(" found finished message "); SYCEXP(ep->msgno);
2597 STRLIT("seconds since last push ");
2598 NPUT(task_now() - ep->start_push, f); STRLIT("ep->client_msg ");
2599 COPY_AND_FREE_GOUT(dbg_pax_msg(ep->client_msg->p)););
2600 IFDBG(D_NONE, FN; STRLIT("ep->p->learner.msg ");
2601 COPY_AND_FREE_GOUT(dbg_pax_msg(ep->p->learner.msg)););
2602
3/4
✓ Branch 0 taken 109702 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 108386 times.
✓ Branch 3 taken 1316 times.
 109702 if (match_my_msg(ep->p->learner.msg, ep->client_msg->p)) {
2603 108386 break;
2604 } else
2605 1316 GOTO(retry_new);
2606 }
2607 108724 next : {
2608
1/2
✓ Branch 0 taken 108724 times.
✗ Branch 1 not taken.
 108724 double now = task_now();
2609 108724 double used = now - ep->start_propose;
2610
1/2
✓ Branch 0 taken 108724 times.
✗ Branch 1 not taken.
 108724 add_to_filter(used);
2611 108724 prop_finished++;
2612 IFDBG(D_NONE, FN; STRLIT("completed ep->msgno "); SYCEXP(ep->msgno);
2613 NDBG(used, f); NDBG(median_time(), f);
2614 STRLIT("seconds since last push "); NDBG(now - ep->start_push, f););
2615 IFDBG(D_NONE, FN; STRLIT("ep->client_msg ");
2616 COPY_AND_FREE_GOUT(dbg_pax_msg(ep->client_msg->p)););
2617 108724 if (ep->p) {
2618 IFDBG(D_NONE, FN; STRLIT("ep->p->learner.msg ");
2619 COPY_AND_FREE_GOUT(dbg_pax_msg(ep->p->learner.msg)););
2620 }
2621
1/2
✓ Branch 0 taken 108724 times.
✗ Branch 1 not taken.
 108724 msg_link_delete(&ep->client_msg);
2622 }
2623 }
2624 ✗ FINALLY
2625 IFDBG(D_BUG, FN; STRLIT("exit "); NDBG(ep->self, d); NDBG(task_now(), f));
2626
2/2
✓ Branch 0 taken 20558 times.
✓ Branch 1 taken 1342 times.
 21900 if (ep->p) {
2627
1/2
✓ Branch 0 taken 20558 times.
✗ Branch 1 not taken.
 20558 unlock_pax_machine(ep->p);
2628 }
2629
1/2
✓ Branch 0 taken 21900 times.
✗ Branch 1 not taken.
 21900 replace_pax_msg(&ep->prepare_msg, nullptr);
2630
2/2
✓ Branch 0 taken 3380 times.
✓ Branch 1 taken 18520 times.
 21900 if (ep->client_msg) { /* If we get here with a client message, we have
2631 failed to deliver */
2632 DBGOUT_ASSERT(check_lsn(ep->client_msg->p->a), STRLIT("NULL lsn"));
2633 IFDBG(D_NONE, FN;
2634 STRLIT("undelivered message at task end -> delivery_failure"));
2635
1/2
✓ Branch 0 taken 3380 times.
✗ Branch 1 not taken.
 3380 deliver_to_app(ep->p, ep->client_msg->p->a, delivery_failure);
2636
1/2
✓ Branch 0 taken 3380 times.
✗ Branch 1 not taken.
 3380 msg_link_delete(&ep->client_msg);
2637 }
2638 21900 remove_proposer_synode(ep->self);
2639
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 21900 times.
✓ Branch 2 taken 21900 times.
✗ Branch 3 not taken.
 21900 TASK_END;
2640 }
2641
2642 static xcom_proto constexpr first_protocol_that_ignores_intermediate_forced_configs_or_views =
2643 x_1_8;
2644
2645 2 static bool constexpr should_ignore_forced_config_or_view(
2646 xcom_proto protocol_version) {
2647 2 return protocol_version >=
2648 2 first_protocol_that_ignores_intermediate_forced_configs_or_views;
2649 }
2650
2651 79660 static node_no get_leader(site_def const *s) {
2652
1/2
✓ Branch 0 taken 79660 times.
✗ Branch 1 not taken.
 79660 if (s) {
2653 79660 node_no leader = 0;
2654
2/2
✓ Branch 0 taken 81955 times.
✓ Branch 1 taken 603 times.
 82558 for (leader = 0; leader < get_maxnodes(s); leader++) {
2655
2/2
✓ Branch 0 taken 79057 times.
✓ Branch 1 taken 2898 times.
 81955 if (!may_be_dead(s->detected, leader, task_now())) return leader;
2656 }
2657 }
2658 603 return 0;
2659 }
2660
2661 79660 int iamthegreatest(site_def const *s) {
2662
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 79660 times.
 79660 if (!s)
2663 ✗ return 0;
2664 else
2665 79660 return get_leader(s) == s->nodeno;
2666 }
2667
2668 // Update site based on incoming global node set
2669 54 static site_def *update_site(site_def *site, node_set const *ns,
2670 synode_no boot_key, synode_no start) {
2671 // If it has not changed, no action is necessary.
2672 // If it has changed, we need to create and install
2673 // a new site def, since the changed node set will influence which
2674 // messages will be ignored.
2675 // This change needs to be effective after the current pipeline
2676 // of messages has been emptied, just as if we had
2677 // changed the config (site_def) itself.
2678
2679
2/2
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 42 times.
 54 if (!equal_node_set(ns, &site->global_node_set)) {
2680 12 site_def *new_config = clone_site_def(get_site_def());
2681
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 12 times.
 12 assert(new_config);
2682 12 new_config->start = start;
2683 12 new_config->boot_key = boot_key;
2684 // Update node set of site
2685 12 copy_node_set(ns, &new_config->global_node_set);
2686 12 return new_config;
2687 }
2688 42 return nullptr;
2689 }
2690
2691 213388 void execute_msg(site_def *site, pax_machine *pma, pax_msg *p) {
2692 213388 app_data_ptr a = p->a;
2693 IFDBG(D_EXEC, FN; COPY_AND_FREE_GOUT(dbg_pax_msg(p)););
2694
1/2
✓ Branch 0 taken 213388 times.
✗ Branch 1 not taken.
 213388 if (a) {
2695
5/5
✓ Branch 0 taken 867 times.
✓ Branch 1 taken 5701 times.
✓ Branch 2 taken 195452 times.
✓ Branch 3 taken 11245 times.
✓ Branch 4 taken 123 times.
 213388 switch (a->body.c_t) {
2696 867 case unified_boot_type:
2697 case force_config_type:
2698 867 deliver_config(a);
2699 6568 case add_node_type:
2700 case remove_node_type:
2701 6568 break;
2702 195452 case app_type:
2703 IFDBG(D_NONE, FN; STRLIT(" learner.msg ");
2704 COPY_AND_FREE_GOUT(dbg_pax_msg(pma->learner.msg)););
2705 /* DBGOUT_ASSERT(check_lsn(a), STRLIT("NULL lsn")); */
2706 195452 deliver_to_app(pma, a, delivery_ok);
2707 195452 break;
2708 11245 case view_msg: {
2709 /* Deliver view like we used to when every member was always a leader.
2710 * This ensures deterministic behaviour in groups with some members
2711 * running previous XCom instances.
2712 */
2713 IFDBG(D_EXEC, FN; STRLIT(" global view ");
2714 COPY_AND_FREE_GOUT(dbg_pax_msg(pma->learner.msg)););
2715
1/2
✓ Branch 0 taken 11245 times.
✗ Branch 1 not taken.
 11245 if (site && site->global_node_set.node_set_len ==
2716
1/2
✓ Branch 0 taken 11245 times.
✗ Branch 1 not taken.
 11245 a->body.app_u_u.present.node_set_len) {
2717
5/6
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 11243 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
✓ Branch 5 taken 11243 times.
 11247 if ((p->force_delivery != 0) &&
2718 2 should_ignore_forced_config_or_view(site->x_proto)) {
2719
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 G_DEBUG(
2720 "execute_msg: Ignoring a forced intermediate, pending "
2721 "view_msg");
2722 } else {
2723
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11243 times.
 11243 assert(site->global_node_set.node_set_len ==
2724 a->body.app_u_u.present.node_set_len);
2725 // Can only mutate site->global_node_set if everyone is a leader and
2726 // has its own channel.
2727
2/2
✓ Branch 0 taken 11185 times.
✓ Branch 1 taken 58 times.
 11243 if (site->max_active_leaders == active_leaders_all) {
2728 11185 copy_node_set(&a->body.app_u_u.present, &site->global_node_set);
2729 }
2730 11243 deliver_global_view_msg(site, a->body.app_u_u.present, p->synode);
2731 ADD_DBG(D_BASE,
2732 add_event(EVENT_DUMP_PAD,
2733 string_arg("deliver_global_view_msg p->synode"));
2734 add_synode_event(p->synode););
2735 }
2736 }
2737
2738 /* If this view_msg is:
2739 *
2740 * (1) about the latest site, and
2741 * (2) only some member(s) is (are) leader(s) in the latest site,
2742 *
2743 * create a new site to deterministically ignore the channel of leaders
2744 * that may be dead. */
2745 11245 site_def *latest_site = get_site_def_rw();
2746 IFDBG(
2747 D_EXEC, FN; PTREXP(latest_site); if (latest_site) {
2748 NUMEXP(latest_site->nodes.node_list_len);
2749 NUMEXP(latest_site->global_node_set.node_set_len);
2750 NUMEXP(a->body.app_u_u.present.node_set_len);
2751 SYCEXP(a->app_key);
2752 SYCEXP(latest_site->start);
2753 });
2754 /*
2755 * You'll want to install the new site if xcom is operating as
2756 * single-leader and there were no changes in the configuration. The
2757 * reason for this is so that you have the latest information about
2758 * who is the preferred and actual leader.
2759 */
2760 11245 bool const is_latest_view = synode_gt(a->app_key, latest_site->start);
2761 11245 bool const everyone_leader_in_latest_site =
2762 11245 (latest_site->max_active_leaders == active_leaders_all);
2763 11245 bool const view_node_set_matches_latest_site =
2764 11245 (latest_site->global_node_set.node_set_len ==
2765 11245 a->body.app_u_u.present.node_set_len);
2766 22466 const bool can_install_site = is_latest_view &&
2767
5/6
✓ Branch 0 taken 11221 times.
✓ Branch 1 taken 24 times.
✓ Branch 2 taken 54 times.
✓ Branch 3 taken 11167 times.
✓ Branch 4 taken 54 times.
✗ Branch 5 not taken.
 11245 !everyone_leader_in_latest_site &&
2768 view_node_set_matches_latest_site;
2769
2770
2/2
✓ Branch 0 taken 54 times.
✓ Branch 1 taken 11191 times.
 11245 if (can_install_site) {
2771 54 a->app_key = p->synode; // Patch app_key to avoid fixing getstart()
2772 54 site_def *new_config = update_site(
2773 54 latest_site, &a->body.app_u_u.present, a->app_key, getstart(a));
2774
2/2
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 42 times.
 54 if (new_config) {
2775 IFDBG(D_EXEC, FN; PTREXP(new_config);
2776 NUMEXP(new_config->nodes.node_list_len);
2777 NUMEXP(new_config->global_node_set.node_set_len);
2778 SYCEXP(a->app_key); SYCEXP(new_config->start););
2779 12 site_install_action(new_config, a->body.c_t);
2780 12 analyze_leaders(new_config);
2781 }
2782 }
2783 11245 } break;
2784 123 default:
2785 123 break;
2786 }
2787 }
2788 IFDBG(D_NONE, FN; SYCEXP(p->synode));
2789 213388 }
2790
2791 static void read_missing_values(int n);
2792 static void propose_missing_values(int n);
2793
2794 #ifdef EXECUTOR_TASK_AGGRESSIVE_NO_OP
2795 /* With many nodes sending read_ops on instances that are not decided yet, it
2796 * may take a very long time until someone finally decides to start a new
2797 * consensus round. As the cost of a new proposal is not that great, it's
2798 * acceptable to go directly to proposing a no-op instead of first trying to
2799 * get the value with a read_op. An added benefit of this is that if more than
2800 * one node needs the result, they will get it all when the consensus round
2801 * finishes. */
2802 static void find_value(site_def const *site, unsigned int *wait, int n) {
2803 IFDBG(D_NONE, FN; NDBG(*wait, d));
2804
2805 if (get_nodeno(site) == VOID_NODE_NO) {
2806 read_missing_values(n);
2807 return;
2808 }
2809
2810 if ((*wait) > 1 || /* Only leader will propose initially */
2811 ((*wait) > 0 && iamthegreatest(site)))
2812 propose_missing_values(n);
2813
2814 #ifdef TASK_EVENT_TRACE
2815 if ((*wait) > 1) dump_task_events();
2816 #endif
2817 (*wait)++;
2818 }
2819 #else
2820 516612 static void find_value(site_def const *site, unsigned int *wait, int n) {
2821 IFDBG(D_NONE, FN; NDBG(*wait, d));
2822
2823
2/2
✓ Branch 0 taken 6993 times.
✓ Branch 1 taken 509619 times.
 516612 if (get_nodeno(site) == VOID_NODE_NO) {
2824 6993 read_missing_values(n);
2825 6993 return;
2826 }
2827
2828
3/4
✓ Branch 0 taken 350061 times.
✓ Branch 1 taken 79058 times.
✓ Branch 2 taken 80500 times.
✗ Branch 3 not taken.
 509619 switch (*wait) {
2829 350061 case 0:
2830 case 1:
2831 350061 read_missing_values(n);
2832 350061 (*wait)++;
2833 350061 break;
2834 79058 case 2:
2835
2/2
✓ Branch 0 taken 48555 times.
✓ Branch 1 taken 30503 times.
 79058 if (iamthegreatest(site))
2836 48555 propose_missing_values(n);
2837 else
2838 30503 read_missing_values(n);
2839 79058 (*wait)++;
2840 79058 break;
2841 80500 case 3:
2842 80500 propose_missing_values(n);
2843 80500 break;
2844 ✗ default:
2845 ✗ break;
2846 }
2847 }
2848 #endif /* EXECUTOR_TASK_AGGRESSIVE_NO_OP */
2849
2850 static void dump_debug_exec_state();
2851
2852 #ifdef PROPOSE_IF_LEADER
2853 int get_xcom_message(pax_machine **p, synode_no msgno, int n) {
2854 DECL_ENV
2855 unsigned int wait;
2856 double delay;
2857 site_def const *site;
2858 ENV_INIT
2859 END_ENV_INIT
2860 END_ENV;
2861
2862 TASK_BEGIN
2863
2864 ep->wait = 0;
2865 ep->delay = 0.0;
2866 *p = force_get_cache(msgno);
2867 ep->site = NULL;
2868
2869 dump_debug_exec_state();
2870 while (!finished(*p)) {
2871 ep->site = find_site_def(msgno);
2872 /* The end of the world ?, fake message by skipping */
2873 if (get_maxnodes(ep->site) == 0) {
2874 pax_msg *msg = pax_msg_new(msgno, ep->site);
2875 handle_skip(ep->site, *p, msg);
2876 break;
2877 }
2878 IFDBG(D_NONE, FN; STRLIT(" not finished "); SYCEXP(msgno); PTREXP(*p);
2879 NDBG(ep->wait, u); SYCEXP(msgno));
2880 if (get_maxnodes(ep->site) > 1 && iamthegreatest(ep->site) &&
2881 ep->site->global_node_set.node_set_val &&
2882 !ep->site->global_node_set.node_set_val[msgno.node] &&
2883 may_be_dead(ep->site->detected, msgno.node, task_now())) {
2884 propose_missing_values(n);
2885 } else {
2886 find_value(ep->site, &ep->wait, n);
2887 }
2888 TIMED_TASK_WAIT(&(*p)->rv, ep->delay = wakeup_delay(ep->delay));
2889 *p = get_cache(msgno);
2890 dump_debug_exec_state();
2891 }
2892
2893 FINALLY
2894 IFDBG(D_NONE, FN; SYCEXP(msgno); PTREXP(*p); NDBG(ep->wait, u);
2895 SYCEXP(msgno));
2896 TASK_END;
2897 }
2898 #else
2899 1109334 int get_xcom_message(pax_machine **p, synode_no msgno, int n) {
2900 DECL_ENV
2901 unsigned int wait;
2902 double delay;
2903 site_def const *site;
2904 592729 ENV_INIT
2905 592729 END_ENV_INIT
2906 END_ENV;
2907
2908
5/9
✓ Branch 0 taken 592729 times.
✓ Branch 1 taken 516605 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 592729 times.
✓ Branch 5 taken 592729 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 592729 times.
 1109334 TASK_BEGIN
2909
2910 592729 ep->wait = 0;
2911 592729 ep->delay = 0.0;
2912 592729 *p = force_get_cache(msgno);
2913 592729 ep->site = nullptr;
2914
2915 592729 dump_debug_exec_state();
2916
2/2
✓ Branch 0 taken 516612 times.
✓ Branch 1 taken 592701 times.
 1109313 while (!finished(*p)) {
2917 516612 ep->site = find_site_def(msgno);
2918 /* The end of the world ?, fake message by skipping */
2919
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 516612 times.
 516612 if (get_maxnodes(ep->site) == 0) {
2920 ✗ pax_msg *msg = pax_msg_new(msgno, ep->site);
2921 ✗ handle_skip(ep->site, *p, msg);
2922 ✗ break;
2923 }
2924 IFDBG(D_NONE, FN; STRLIT("before find_value"); SYCEXP(msgno); PTREXP(*p);
2925 NDBG(ep->wait, u); SYCEXP(msgno));
2926 516612 find_value(ep->site, &ep->wait, n);
2927 IFDBG(D_NONE, FN; STRLIT("after find_value"); SYCEXP(msgno); PTREXP(*p);
2928 NDBG(ep->wait, u); SYCEXP(msgno));
2929 516612 ep->delay = wakeup_delay(ep->delay);
2930 IFDBG(D_NONE, FN; NDBG(ep->delay, f));
2931
4/6
✗ Branch 0 not taken.
✓ Branch 1 taken 516605 times.
✓ Branch 2 taken 516605 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 21 times.
✓ Branch 5 taken 516584 times.
 1033217 TIMED_TASK_WAIT(&(*p)->rv, ep->delay);
2932 516584 *p = get_cache(msgno);
2933 516584 dump_debug_exec_state();
2934 }
2935
2936 592701 FINALLY
2937
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 592722 times.
 592722 TASK_END;
2938 }
2939 #endif
2940
2941 598149 synode_no set_executed_msg(synode_no msgno) {
2942 IFDBG(D_EXEC, FN; STRLIT("changing executed_msg from "); SYCEXP(executed_msg);
2943 STRLIT(" to "); SYCEXP(msgno));
2944
3/4
✓ Branch 0 taken 598149 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 211591 times.
✓ Branch 3 taken 386558 times.
 1196298 if (group_mismatch(msgno, current_message) ||
2945
2/2
✓ Branch 0 taken 211591 times.
✓ Branch 1 taken 386558 times.
 598149 synode_gt(msgno, current_message)) {
2946 IFDBG(D_EXEC, FN; STRLIT("changing current message"));
2947 211591 set_current_message(first_free_synode_local(msgno));
2948 }
2949
2950
2/2
✓ Branch 0 taken 301539 times.
✓ Branch 1 taken 296610 times.
 598149 if (msgno.msgno > executed_msg.msgno) task_wakeup(&exec_wait);
2951
2952 598149 executed_msg = msgno;
2953 598149 executor_site = find_site_def_rw(executed_msg);
2954 598149 return executed_msg;
2955 }
2956
2957 212951 static synode_no first_free_synode_local(synode_no msgno) {
2958
1/2
✓ Branch 0 taken 212951 times.
✗ Branch 1 not taken.
 212951 site_def const *site = find_site_def(msgno);
2959 212951 synode_no retval = msgno;
2960
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 212951 times.
 212951 if (!site) {
2961 /* purecov: begin deadcode */
2962 ✗ site = get_site_def();
2963 IFDBG(D_NONE, FN; PTREXP(site); SYCEXP(msgno));
2964 ✗ assert(get_group_id(site) != 0);
2965 /* purecov: end */
2966 }
2967
1/2
✓ Branch 0 taken 212951 times.
✗ Branch 1 not taken.
 212951 if (get_group_id(site) == 0) {
2968 IFDBG(D_NONE, FN; PTREXP(site); SYCEXP(msgno));
2969 if (site) {
2970 IFDBG(D_NONE, FN; SYCEXP(site->boot_key); SYCEXP(site->start);
2971 COPY_AND_FREE_GOUT(dbg_site_def(site)));
2972 }
2973 }
2974
2/4
✓ Branch 0 taken 212951 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 212951 times.
 212951 assert(get_group_id(site) != 0);
2975
2/4
✓ Branch 0 taken 212951 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 212951 times.
 212951 assert(!synode_eq(msgno, null_synode));
2976
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 212951 times.
 212951 if (retval.msgno == 0) retval.msgno = 1;
2977
1/2
✓ Branch 0 taken 212951 times.
✗ Branch 1 not taken.
 212951 retval.node = get_nodeno(site);
2978
3/4
✓ Branch 0 taken 212951 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 60051 times.
✓ Branch 3 taken 152900 times.
 212951 if (synode_lt(retval, msgno))
2979
1/2
✓ Branch 0 taken 60051 times.
✗ Branch 1 not taken.
 60051 return incr_msgno(retval);
2980 else
2981 152900 return retval;
2982 }
2983
2984 323655 synode_no set_current_message(synode_no msgno) {
2985 IFDBG(D_PROPOSE, FN; STRLIT("changing current_message from ");
2986 SYCEXP(current_message); STRLIT(" to "); SYCEXP(msgno));
2987 323655 return current_message = msgno;
2988 }
2989
2990 static void update_max_synode(pax_msg *p);
2991
2992 #if TASK_DBUG_ON
2993 static void perf_dbg(int *_n, int *_old_n, double *_old_t) [[maybe_unused]];
2994 static void perf_dbg(int *_n, int *_old_n, double *_old_t) {
2995 int n = *_n;
2996 int old_n = *_old_n;
2997 double old_t = *_old_t;
2998
2999 if (!IS_XCOM_DEBUG_WITH(XCOM_DEBUG_TRACE)) return;
3000
3001 IFDBG(D_NONE, FN; SYCEXP(executed_msg));
3002 if (!(n % 5000)) {
3003 GET_GOUT;
3004 NDBG(get_nodeno(get_site_def()), u);
3005 NDBG(task_now(), f);
3006 NDBG(n, d);
3007 NDBG(median_time(), f);
3008 SYCEXP(executed_msg);
3009 PRINT_GOUT;
3010 FREE_GOUT;
3011 }
3012 (*_n)++;
3013 if (task_now() - old_t > 1.0) {
3014 GET_GOUT;
3015 NDBG(get_nodeno(get_site_def()), u);
3016 NDBG(task_now(), f);
3017 NDBG(n, d);
3018 NDBG((n - old_n) / (task_now() - old_t), f);
3019 PRINT_GOUT;
3020 FREE_GOUT;
3021 *_old_t = task_now();
3022 *_old_n = n;
3023 }
3024 }
3025 #endif
3026
3027 /* Does address match any current leader? */
3028 80 static inline int match_leader(char const *addr, leader_array const leaders) {
3029 u_int i;
3030
2/2
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 24 times.
 104 for (i = 0; i < leaders.leader_array_len; i++) {
3031 IFDBG(D_BASE, FN; NUMEXP(i); NUMEXP(leaders.leader_array_len); STREXP(addr);
3032 STREXP(leaders.leader_array_val[i].address));
3033
2/2
✓ Branch 0 taken 56 times.
✓ Branch 1 taken 24 times.
 80 if (strcmp(addr, leaders.leader_array_val[i].address) == 0) return 1;
3034 }
3035 24 return 0;
3036 }
3037
3038 104 static inline bool alive_node(site_def const *site, u_int i) {
3039 104 return is_set(site->global_node_set, i);
3040 }
3041
3042 // Find up to site->max_active_leaders leaders.
3043 // If leaders are set by the client, and none of those are alive, revert
3044 // to using the set of addresses in the config.
3045
3046 71 void analyze_leaders(site_def *site) {
3047
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 71 times.
 71 assert(site);
3048 // No analysis if all nodes are leaders
3049
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 68 times.
 71 if (active_leaders_all == site->max_active_leaders) return;
3050
3051 // Use leaders from config if forced or not set by client
3052 68 bool use_client_leaders = leaders_set_by_client(site);
3053 68 site->cached_leaders = true;
3054 68 site->found_leaders = 0; // Number of active leaders found
3055 // Reset everything
3056
2/2
✓ Branch 0 taken 141 times.
✓ Branch 1 taken 68 times.
 209 for (u_int i = 0; i < get_maxnodes(site); i++) {
3057 141 site->active_leader[i] = 0;
3058 }
3059 // If candidate leaders set by client, check those first
3060
6/6
✓ Branch 0 taken 197 times.
✓ Branch 1 taken 6 times.
✓ Branch 2 taken 135 times.
✓ Branch 3 taken 62 times.
✓ Branch 4 taken 135 times.
✓ Branch 5 taken 68 times.
 203 for (u_int i = 0; use_client_leaders && i < get_maxnodes(site); i++) {
3061 270 if (site->found_leaders <
3062 224 site->max_active_leaders && // Found enough?
3063 // Must be alive according to global
3064 // node set of site
3065
8/8
✓ Branch 0 taken 89 times.
✓ Branch 1 taken 46 times.
✓ Branch 2 taken 80 times.
✓ Branch 3 taken 9 times.
✓ Branch 4 taken 56 times.
✓ Branch 5 taken 24 times.
✓ Branch 6 taken 56 times.
✓ Branch 7 taken 79 times.
 215 alive_node(site, i) &&
3066 // Must match a node in the list of leaders
3067 80 match_leader(site->nodes.node_list_val[i].address, site->leaders)) {
3068 56 site->active_leader[i] = 1;
3069 56 site->found_leaders++;
3070 }
3071 }
3072 // Check rest of nodes
3073
2/2
✓ Branch 0 taken 141 times.
✓ Branch 1 taken 68 times.
 209 for (u_int i = 0; i < get_maxnodes(site); i++) {
3074 367 if (!site->active_leader[i] && // Avoid duplicates
3075
6/6
✓ Branch 0 taken 85 times.
✓ Branch 1 taken 56 times.
✓ Branch 2 taken 15 times.
✓ Branch 3 taken 70 times.
✓ Branch 4 taken 6 times.
✓ Branch 5 taken 135 times.
 156 site->found_leaders < site->max_active_leaders && // Found enough?
3076 // Must be alive according to global
3077 // node set of site
3078
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 9 times.
 15 alive_node(site, i)) {
3079 6 site->active_leader[i] = 1;
3080 6 site->found_leaders++;
3081 }
3082 }
3083 // We need at least one channel otherwise the group grinds to a halt.
3084
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 62 times.
 68 if (site->found_leaders == 0) {
3085 6 site->active_leader[0] = 1;
3086 6 site->found_leaders = 1;
3087 }
3088 68 free(site->dispatch_table);
3089
3090 IFDBG(D_BUG, FN; STRLIT("free "); PTREXP(site); PTREXP(site->dispatch_table));
3091 // Do not work as synode allocator if not active leader. ???
3092
4/4
✓ Branch 0 taken 59 times.
✓ Branch 1 taken 9 times.
✓ Branch 2 taken 33 times.
✓ Branch 3 taken 35 times.
 127 if (get_nodeno(site) != VOID_NODE_NO &&
3093
2/2
✓ Branch 0 taken 33 times.
✓ Branch 1 taken 26 times.
 59 site->active_leader[get_nodeno(site)]) {
3094 33 site->dispatch_table = primary_dispatch_table();
3095 } else {
3096 35 site->dispatch_table = secondary_dispatch_table();
3097 }
3098 IFDBG(D_BUG, FN; STRLIT("allocate "); PTREXP(site);
3099 PTREXP(site->dispatch_table));
3100
3101
2/2
✓ Branch 0 taken 141 times.
✓ Branch 1 taken 68 times.
 209 for (u_int i = 0; i < get_maxnodes(site); i++) {
3102 IFDBG(D_BUG, FN; NUMEXP(i); PTREXP(site); NUMEXP(site->found_leaders);
3103 NUMEXP(site->max_active_leaders); NUMEXP(alive_node(site, i));
3104 SYCEXP(site->start); STREXP(site->nodes.node_list_val[i].address);
3105 if (site->active_leader[i]) STRLIT(" says YES");
3106 else STRLIT(" says NO"));
3107 }
3108 }
3109
3110 /* Is node number an active leader? */
3111 1669005 int is_active_leader(node_no x, site_def *site) {
3112 /* No site, no active leaders */
3113
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1669005 times.
 1669005 if (!site) return 0;
3114
3115 /* Node number out of bound, not an active leader */
3116
2/2
✓ Branch 0 taken 6754 times.
✓ Branch 1 taken 1662251 times.
 1669005 if (x >= get_maxnodes(site)) return 0;
3117
3118 /* All are leaders, no need for further tests */
3119
2/2
✓ Branch 0 taken 1654347 times.
✓ Branch 1 taken 7904 times.
 1662251 if (active_leaders_all == site->max_active_leaders) return 1;
3120 /* See if cached values are valid */
3121
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 7863 times.
 7904 if (!site->cached_leaders) {
3122 41 analyze_leaders(site);
3123 }
3124 #if 0
3125 if (site->active_leader == NULL || x > site->nodes.node_list_len - 1)
3126 IFDBG(D_BUG, FN; PTREXP(site->active_leader); NUMEXP(x);
3127 NUMEXP(site->nodeno); NUMEXP(site->nodes.node_list_len););
3128 #endif
3129 7904 return site->active_leader[x];
3130 }
3131
3132 18 node_no found_active_leaders(site_def *site) {
3133 /* No site, no active leaders */
3134
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 18 times.
 18 if (!site) return 0;
3135
3136 /* All are leaders, no need for further tests */
3137
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 15 times.
 18 if (active_leaders_all == site->max_active_leaders)
3138 3 return site->nodes.node_list_len;
3139
3140 /* See if cached values are valid */
3141
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 15 times.
 15 if (!site->cached_leaders) {
3142 ✗ analyze_leaders(site);
3143 }
3144 15 return site->found_leaders;
3145 }
3146
3147 /* Check if this message belongs to a channel that should be ignored */
3148 1505672 static inline int ignore_message(synode_no x, site_def *site,
3149 char const *dbg [[maybe_unused]]) {
3150 1505672 int retval = !is_active_leader(x.node, site);
3151 IFDBG(D_BASE, STRLIT(dbg); STRLIT(" "); FN; SYCEXP(x); NUMEXP(retval));
3152 1505672 return retval;
3153 }
3154
3155 /* Check if this node is a leader */
3156 113893 static inline bool is_leader(site_def *site) {
3157
3/4
✓ Branch 0 taken 113893 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 110264 times.
✓ Branch 3 taken 3629 times.
 113893 bool retval = site && is_active_leader(site->nodeno, site);
3158 IFDBG(D_BASE, FN; PTREXP(site); if (site) NUMEXP(site->nodeno);
3159 NUMEXP(retval));
3160 113893 return retval;
3161 }
3162
3163 [[maybe_unused]] static void debug_loser(synode_no x);
3164 #if defined(TASK_DBUG_ON) && TASK_DBUG_ON
3165 static void debug_loser(synode_no x) {
3166 if (!IS_XCOM_DEBUG_WITH(XCOM_DEBUG_TRACE)) return;
3167 if (1 || x.msgno < 10) {
3168 GET_GOUT;
3169 NDBG(get_nodeno(find_site_def(x)), u);
3170 STRLIT(" ignoring loser ");
3171 SYCEXP(x);
3172 SYCEXP(max_synode);
3173 PRINT_GOUT;
3174 FREE_GOUT;
3175 }
3176 }
3177 #else
3178 /* purecov: begin deadcode */
3179 ✗ static void debug_loser(synode_no x [[maybe_unused]]) {}
3180 /* purecov: end */
3181 #endif
3182
3183 32785 static void send_value(site_def const *site, node_no to, synode_no synode) {
3184 32785 pax_machine *pm = get_cache(synode);
3185
3/4
✓ Branch 0 taken 32785 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 30083 times.
✓ Branch 3 taken 2702 times.
 32785 if (pm && pm->learner.msg) {
3186
1/2
✓ Branch 0 taken 30083 times.
✗ Branch 1 not taken.
 30083 pax_msg *msg = clone_pax_msg(pm->learner.msg);
3187
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 30083 times.
 30083 if (msg == nullptr) return;
3188
1/2
✓ Branch 0 taken 30083 times.
✗ Branch 1 not taken.
 30083 ref_msg(msg);
3189
1/2
✓ Branch 0 taken 30083 times.
✗ Branch 1 not taken.
 30083 send_server_msg(site, to, msg);
3190
1/2
✓ Branch 0 taken 30083 times.
✗ Branch 1 not taken.
 30083 unref_msg(&msg);
3191 }
3192 }
3193
3194 /**
3195 * Returns the message number where it is safe for nodes in previous
3196 * configuration to exit.
3197 *
3198 * @param start start synod of the next configuration
3199 * @param event_horizon event horizon of the next configuration
3200 */
3201 8407 static synode_no compute_delay(synode_no start,
3202 xcom_event_horizon event_horizon) {
3203 8407 start.msgno += event_horizon;
3204 8407 return start;
3205 }
3206
3207 /* Push messages to all nodes which were in the previous site, but not in this
3208 */
3209 8830 static void inform_removed(int index, int all) {
3210 8830 site_def **sites = nullptr;
3211 8830 uint32_t site_count = 0;
3212 IFDBG(D_NONE, FN; NEXP(index, d));
3213
1/2
✓ Branch 0 taken 8830 times.
✗ Branch 1 not taken.
 8830 get_all_site_defs(&sites, &site_count);
3214
4/6
✓ Branch 0 taken 8830 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6661 times.
✓ Branch 3 taken 2169 times.
✓ Branch 4 taken 6661 times.
✗ Branch 5 not taken.
 8830 while (site_count > 1 && index >= 0 && (uint32_t)(index + 1) < site_count) {
3215 6661 site_def *s = sites[index];
3216 6661 site_def *ps = sites[index + 1];
3217
3218 /* Compute diff and push messages */
3219 IFDBG(D_NONE, FN; NDBG(index, d); PTREXP(s); if (s) SYCEXP(s->boot_key);
3220 PTREXP(ps); if (ps) SYCEXP(ps->boot_key));
3221
3222
2/4
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6661 times.
✗ Branch 3 not taken.
 6661 if (s && ps) {
3223 6661 node_no i = 0;
3224 IFDBG(D_NONE, FN; SYCEXP(s->boot_key); SYCEXP(s->start);
3225 SYCEXP(ps->boot_key); SYCEXP(ps->start));
3226
2/2
✓ Branch 0 taken 12334 times.
✓ Branch 1 taken 6661 times.
 18995 for (i = 0; i < ps->nodes.node_list_len; i++) { /* Loop over prev site */
3227
4/4
✓ Branch 0 taken 5673 times.
✓ Branch 1 taken 6661 times.
✓ Branch 2 taken 1783 times.
✓ Branch 3 taken 10551 times.
 18007 if (ps->nodeno != i &&
3228
3/4
✓ Branch 0 taken 5673 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1783 times.
✓ Branch 3 taken 3890 times.
 5673 !node_exists(&ps->nodes.node_list_val[i], &s->nodes)) {
3229 1783 synode_no synode = s->start;
3230 1783 synode_no end = max_synode;
3231
3/4
✓ Branch 0 taken 34568 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 32785 times.
✓ Branch 3 taken 1783 times.
 34568 while (!synode_gt(synode, end)) { /* Loop over relevant messages */
3232
1/2
✓ Branch 0 taken 32785 times.
✗ Branch 1 not taken.
 32785 send_value(ps, i, synode);
3233
1/2
✓ Branch 0 taken 32785 times.
✗ Branch 1 not taken.
 32785 synode = incr_synode(synode);
3234 }
3235 }
3236 }
3237 }
3238
1/2
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
 6661 if (!all) /* Early exit if not all configs should be examined */
3239 6661 break;
3240 ✗ index--;
3241 }
3242 8830 }
3243
3244 ✗ static bool_t backwards_compatible(xcom_event_horizon event_horizon) {
3245 ✗ return event_horizon == EVENT_HORIZON_MIN;
3246 }
3247
3248 static xcom_proto const first_event_horizon_aware_protocol = x_1_4;
3249
3250 3448 static bool_t reconfigurable_event_horizon(xcom_proto protocol_version) {
3251 3448 return protocol_version >= first_event_horizon_aware_protocol;
3252 }
3253
3254 1673 static bool_t add_node_unsafe_against_ipv4_old_nodes(app_data_ptr a) {
3255
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1673 times.
 1673 assert(a->body.c_t == add_node_type);
3256
3257 {
3258 1673 site_def const *latest_config = get_site_def();
3259
3/6
✓ Branch 0 taken 1673 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1673 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1673 times.
✗ Branch 5 not taken.
 1673 if (latest_config && latest_config->x_proto >= minimum_ipv6_version())
3260 1673 return FALSE;
3261
3262 {
3263 ✗ u_int const nr_nodes_to_add = a->body.app_u_u.nodes.node_list_len;
3264 ✗ node_address *nodes_to_add = a->body.app_u_u.nodes.node_list_val;
3265
3266 u_int i;
3267 ✗ xcom_port node_port = 0;
3268 char node_addr[IP_MAX_SIZE];
3269
3270 ✗ for (i = 0; i < nr_nodes_to_add; i++) {
3271 ✗ if (get_ip_and_port(nodes_to_add[i].address, node_addr, &node_port)) {
3272 ✗ G_ERROR(
3273 "Error parsing address from a joining node. Join operation "
3274 "will be "
3275 "rejected");
3276 ✗ return TRUE;
3277 }
3278
3279 ✗ if (!is_node_v4_reachable(node_addr)) return TRUE;
3280 }
3281 }
3282
3283 ✗ return FALSE;
3284 }
3285 }
3286
3287 /**
3288 * @brief This will test if we are receiving a boot request that contains
3289 * ourselves. This could happed in case of a misconfiguration of a
3290 * local_address, that causes an add_node request to be erroneous
3291 * delivered.
3292 *
3293 * @param a app_data with an add node request
3294 * @return bool_t TRUE in case of error, meaning that my address is in the
3295 * add_node list
3296 */
3297 ✗ static bool_t add_node_adding_own_address(app_data_ptr a) {
3298 ✗ assert(a->body.c_t == add_node_type);
3299
3300 ✗ return node_exists(cfg_app_xcom_get_identity(), &a->body.app_u_u.nodes);
3301 }
3302
3303 /**
3304 * Check if a node is compatible with the group's event horizon.
3305 *
3306 * A node is compatible with the group's configuration if:
3307 *
3308 * a) The node supports event horizon reconfigurations, or
3309 * b) The group's event horizon is, or is scheduled to be, the default
3310 * event horizon.
3311 */
3312 3420 static bool unsafe_against_event_horizon(node_address const *node) {
3313 3420 site_def const *latest_config = get_site_def();
3314 3420 xcom_proto node_max_protocol_version = node->proto.max_proto;
3315 bool const compatible =
3316
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 3420 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 3420 reconfigurable_event_horizon(node_max_protocol_version) ||
3317 ✗ backwards_compatible(latest_config->event_horizon);
3318
3319
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3420 times.
 3420 if (!compatible) {
3320 /*
3321 * The node that wants to join does not support event horizon
3322 * reconfigurations and the group's event horizon is, or is scheduled to
3323 * be, different from the default.
3324 * The node can not safely join the group so we deny its attempt to join.
3325 */
3326 ✗ G_INFO(
3327 "%s's request to join the group was rejected because the group's "
3328 "event "
3329 "horizon is, or will be %" PRIu32 " and %s only supports %" PRIu32,
3330 node->address, latest_config->event_horizon, node->address,
3331 EVENT_HORIZON_MIN);
3332 ✗ return true;
3333 }
3334 3420 return false;
3335 }
3336
3337 typedef bool (*unsafe_node_check)(node_address const *node);
3338
3339 10308 static bool check_if_add_node_is_unsafe(app_data_ptr a,
3340 unsafe_node_check unsafe) {
3341
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 10308 times.
 10308 assert(a->body.c_t == add_node_type);
3342 {
3343 10308 u_int nodes_len = a->body.app_u_u.nodes.node_list_len;
3344 10308 node_address *nodes_to_add = a->body.app_u_u.nodes.node_list_val;
3345 u_int i;
3346
2/2
✓ Branch 0 taken 10302 times.
✓ Branch 1 taken 10302 times.
 20604 for (i = 0; i < nodes_len; i++) {
3347
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 10296 times.
 10302 if (unsafe(&nodes_to_add[i])) return true;
3348 }
3349 }
3350 10302 return false;
3351 }
3352
3353 3420 static bool check_if_add_node_is_unsafe_against_event_horizon(app_data_ptr a) {
3354 3420 return check_if_add_node_is_unsafe(a, unsafe_against_event_horizon);
3355 }
3356
3357 // Map values of old node set to new node set by matching on
3358 // node addresses
3359 13145 void recompute_node_set(node_set const *old_set, node_list const *old_nodes,
3360 node_set *new_set, node_list const *new_nodes) {
3361 // Return value of node set matching node_address na
3362 19662 auto value{[&](node_address const *na) {
3363
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19662 times.
 19662 assert(old_set->node_set_len == old_nodes->node_list_len);
3364
2/2
✓ Branch 0 taken 31056 times.
✓ Branch 1 taken 3494 times.
 34550 for (u_int i = 0; i < old_nodes->node_list_len; i++) {
3365
2/2
✓ Branch 0 taken 16168 times.
✓ Branch 1 taken 14888 times.
 31056 if (match_node(&old_nodes->node_list_val[i], na, true)) {
3366 16168 return old_set->node_set_val[i];
3367 }
3368 }
3369 3494 return 0;
3370 13145 }};
3371
3372
2/2
✓ Branch 0 taken 19662 times.
✓ Branch 1 taken 13145 times.
 32807 for (u_int i = 0; i < new_nodes->node_list_len; i++) {
3373
1/2
✓ Branch 0 taken 19662 times.
✗ Branch 1 not taken.
 19662 new_set->node_set_val[i] = value(&new_nodes->node_list_val[i]);
3374 }
3375 13145 }
3376
3377 // Remap old global and local node set of site to new
3378 6568 static void recompute_node_sets(site_def const *old_site, site_def *new_site) {
3379 6568 recompute_node_set(&old_site->global_node_set, &old_site->nodes,
3380 6568 &new_site->global_node_set, &new_site->nodes);
3381 6568 recompute_node_set(&old_site->local_node_set, &old_site->nodes,
3382 6568 &new_site->local_node_set, &new_site->nodes);
3383 6568 }
3384
3385 3608 static bool incompatible_proto_and_max_leaders(xcom_proto x_proto,
3386 node_no max_leaders) {
3387
4/4
✓ Branch 0 taken 21 times.
✓ Branch 1 taken 3587 times.
✓ Branch 2 taken 12 times.
✓ Branch 3 taken 9 times.
 3608 return x_proto < single_writer_support && max_leaders != active_leaders_all;
3388 }
3389
3390 3632 static bool incompatible_proto_and_leaders(xcom_proto x_proto) {
3391 3632 return x_proto < single_writer_support;
3392 }
3393
3394 3444 static bool incompatible_proto_and_max_leaders(node_address const *node) {
3395 3444 site_def const *latest_config = get_site_def();
3396
3397
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 3438 times.
 3444 if (incompatible_proto_and_max_leaders(node->proto.max_proto,
3398 3444 latest_config->max_active_leaders)) {
3399 /*
3400 * The node that wants to join does not allow setting of max number of
3401 * leaders
3402 * and the max number of leaders in the group is not all.
3403 * The node can not safely join the group so we deny its attempt to join.
3404 */
3405
2/4
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
 6 G_INFO(
3406 "%s's request to join the group was rejected because the group's max "
3407 "number of active leaders is, or will be %" PRIu32
3408 " and %s only supports "
3409 "all nodes as leaders",
3410 node->address, latest_config->max_active_leaders, node->address);
3411 6 return true;
3412 }
3413 3438 return false;
3414 }
3415
3416 3438 static bool incompatible_proto_and_leaders(node_address const *node) {
3417 3438 site_def const *latest_config = get_site_def();
3418
3419
4/6
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 3432 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 6 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 3438 times.
 3444 if (incompatible_proto_and_leaders(node->proto.max_proto) &&
3420 6 leaders_set_by_client(latest_config)) {
3421 /*
3422 * The node that wants to join does not allow changing the set of
3423 * leaders
3424 * and the set of leaders in the group is not empty.
3425 * The node can not safely join the group so we deny its attempt to join.
3426 */
3427 ✗ G_INFO(
3428 "%s's request to join the group was rejected because the group "
3429 "has a non-empty set of leaders specified by the client, "
3430 "and %s does not support changing the set of leaders",
3431 node->address, node->address);
3432 ✗ return true;
3433 }
3434 3438 return false;
3435 }
3436
3437 3447 bool unsafe_leaders(app_data *a) {
3438
2/2
✓ Branch 0 taken 3441 times.
✓ Branch 1 taken 6 times.
 6888 return check_if_add_node_is_unsafe(a, incompatible_proto_and_max_leaders) ||
3439
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3441 times.
 6888 check_if_add_node_is_unsafe(a, incompatible_proto_and_leaders);
3440 }
3441
3442 5955 static void set_start_and_boot(site_def *new_config, app_data_ptr a) {
3443 5955 new_config->start = getstart(a);
3444 5955 new_config->boot_key = a->app_key;
3445 5955 }
3446
3447 // Map values of old timestamps to new timestamps by matching on
3448 // node addresses
3449 6577 void recompute_timestamps(detector_state const old_timestamp,
3450 node_list const *old_nodes,
3451 detector_state new_timestamp,
3452 node_list const *new_nodes) {
3453 // Return value of timestamp matching node_address na
3454 9837 auto value{[&](node_address const *na) {
3455
2/2
✓ Branch 0 taken 15537 times.
✓ Branch 1 taken 1747 times.
 17284 for (u_int i = 0; i < old_nodes->node_list_len; i++) {
3456
2/2
✓ Branch 0 taken 8090 times.
✓ Branch 1 taken 7447 times.
 15537 if (match_node(&old_nodes->node_list_val[i], na, true)) {
3457 8090 return old_timestamp[i];
3458 }
3459 }
3460 1747 return 0.0;
3461 6577 }};
3462
3463
2/2
✓ Branch 0 taken 9837 times.
✓ Branch 1 taken 6577 times.
 16414 for (u_int i = 0; i < new_nodes->node_list_len; i++) {
3464
1/2
✓ Branch 0 taken 9837 times.
✗ Branch 1 not taken.
 9837 new_timestamp[i] = value(&new_nodes->node_list_val[i]);
3465 }
3466 6577 }
3467
3468 /**
3469 * Reconfigure the group membership: add new member(s).
3470 *
3471 * It is possible that concurrent reconfigurations take effect between the
3472 * time this reconfiguration was proposed and now.
3473 *
3474 * Particularly, it is possible that any of the concurrent reconfigurations
3475 * modified the event horizon and that the new member(s) do not support event
3476 * horizon reconfigurations.
3477 *
3478 * We account for these situations by validating if adding the new members is
3479 * still possible under the current state.
3480 *
3481 * If it is not, this reconfiguration does not produce any effect, i.e. no new
3482 * configuration is installed.
3483 */
3484 1747 site_def *handle_add_node(app_data_ptr a) {
3485
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1747 times.
 1747 if (check_if_add_node_is_unsafe_against_event_horizon(a)) {
3486 /*
3487 * Note that the result of this function is only applicable to
3488 * unused and not-fully-implemented code paths where add_node_type is used
3489 * forcibly.
3490 * Should this fact change, this obviously does not work.
3491 */
3492 ✗ return nullptr;
3493 }
3494
3495
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1747 times.
 1747 if (unsafe_leaders(a)) {
3496 ✗ return nullptr;
3497 }
3498 {
3499
2/2
✓ Branch 0 taken 1747 times.
✓ Branch 1 taken 1747 times.
 3494 for (u_int node = 0; node < a->body.app_u_u.nodes.node_list_len; node++) {
3500
2/4
✓ Branch 0 taken 1747 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1747 times.
✗ Branch 3 not taken.
 1747 G_INFO("Adding new node to the configuration: %s",
3501 a->body.app_u_u.nodes.node_list_val[node].address)
3502 }
3503
3504 1747 site_def const *old_site = get_site_def();
3505 1747 site_def *site = clone_site_def(old_site);
3506 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&a->body.app_u_u.nodes)););
3507 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&a->body.app_u_u.nodes)););
3508 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("a->app_key"));
3509 add_synode_event(a->app_key););
3510
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1747 times.
 1747 assert(old_site);
3511
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1747 times.
 1747 assert(site);
3512 1747 add_site_def(a->body.app_u_u.nodes.node_list_len,
3513 a->body.app_u_u.nodes.node_list_val, site);
3514 1747 set_start_and_boot(site, a);
3515
1/2
✓ Branch 0 taken 1747 times.
✗ Branch 1 not taken.
 1747 if (site->x_proto >= single_writer_support) {
3516 1747 recompute_node_sets(old_site, site);
3517 1747 recompute_timestamps(old_site->detected, &old_site->nodes, site->detected,
3518 1747 &site->nodes);
3519 }
3520 1747 site_install_action(site, a->body.c_t);
3521 1747 return site;
3522 }
3523 }
3524
3525 /**
3526 * Check if we can reconfigure the event horizon.
3527 *
3528 * We can reconfigure the event horizon if all group members support
3529 * reconfiguring the event horizon, and the new event horizon in the domain
3530 * [EVENT_HORIZON_MIN, EVENT_HORIZON_MAX].
3531 *
3532 * We use the group's latest common XCom protocol as a proxy to decide if all
3533 * members support reconfiguring the event horizon.
3534 *
3535 * If the common protocol is at least version 5 (x_1_4) then all members run
3536 * compatible server instances.
3537 *
3538 * Otherwise there are older instances, and it follows that the event horizon
3539 * must be the default and cannot be reconfigured.
3540 */
3541 enum allow_event_horizon_result {
3542 EVENT_HORIZON_ALLOWED,
3543 EVENT_HORIZON_INVALID,
3544 EVENT_HORIZON_UNCHANGEABLE
3545 };
3546 typedef enum allow_event_horizon_result allow_event_horizon_result;
3547
3548 ✗ static void log_event_horizon_reconfiguration_failure(
3549 allow_event_horizon_result error_code,
3550 xcom_event_horizon attempted_event_horizon) {
3551 ✗ switch (error_code) {
3552 ✗ case EVENT_HORIZON_INVALID:
3553 ✗ G_WARNING("The event horizon was not reconfigured to %" PRIu32
3554 "because its domain is [%" PRIu32 ", %" PRIu32 "]",
3555 attempted_event_horizon, xcom_get_minimum_event_horizon(),
3556 xcom_get_maximum_event_horizon());
3557 ✗ break;
3558 ✗ case EVENT_HORIZON_UNCHANGEABLE:
3559 ✗ G_WARNING("The event horizon was not reconfigured to %" PRIu32
3560 " because some of the group's members do not support "
3561 "reconfiguring the event horizon",
3562 attempted_event_horizon);
3563 ✗ break;
3564 ✗ case EVENT_HORIZON_ALLOWED:
3565 ✗ break;
3566 }
3567 }
3568
3569 28 static allow_event_horizon_result allow_event_horizon(
3570 xcom_event_horizon event_horizon) {
3571
2/4
✓ Branch 0 taken 28 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 28 times.
 28 if (event_horizon < EVENT_HORIZON_MIN || event_horizon > EVENT_HORIZON_MAX)
3572 ✗ return EVENT_HORIZON_INVALID;
3573
3574 {
3575 28 const site_def *latest_config = get_site_def();
3576
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 if (!reconfigurable_event_horizon(latest_config->x_proto)) {
3577 ✗ assert(backwards_compatible(latest_config->event_horizon));
3578 ✗ return EVENT_HORIZON_UNCHANGEABLE;
3579 }
3580 }
3581 28 return EVENT_HORIZON_ALLOWED;
3582 }
3583
3584 28 static bool_t is_unsafe_event_horizon_reconfiguration(app_data_ptr a) {
3585
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 assert(a->body.c_t == set_event_horizon_type);
3586 {
3587 28 xcom_event_horizon new_event_horizon = a->body.app_u_u.event_horizon;
3588 28 bool_t result = FALSE;
3589 allow_event_horizon_result error_code;
3590 28 error_code = allow_event_horizon(new_event_horizon);
3591
1/3
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 switch (error_code) {
3592 ✗ case EVENT_HORIZON_INVALID:
3593 case EVENT_HORIZON_UNCHANGEABLE:
3594 ✗ log_event_horizon_reconfiguration_failure(error_code,
3595 new_event_horizon);
3596 ✗ result = TRUE;
3597 ✗ break;
3598 28 case EVENT_HORIZON_ALLOWED:
3599 28 break;
3600 }
3601 28 return result;
3602 }
3603 }
3604
3605 // Predicate that checks IF the reconfiguration will be unsafe
3606 182 static bool_t is_unsafe_max_leaders_reconfiguration(app_data_ptr a) {
3607
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 182 times.
 182 assert(a->body.c_t == set_max_leaders);
3608 182 const site_def *latest_config = get_site_def();
3609 182 node_no new_max_leaders = a->body.app_u_u.max_leaders;
3610
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 164 times.
 182 if (new_max_leaders > get_maxnodes(latest_config)) {
3611
3/6
✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 18 times.
✗ Branch 5 not taken.
 18 G_WARNING("The max number of leaders was not reconfigured to %" PRIu32
3612 " because its domain is [%" PRIu32 ", %" PRIu32 "]",
3613 new_max_leaders, 0, get_maxnodes(latest_config));
3614 18 return TRUE;
3615
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 158 times.
 164 } else if (incompatible_proto_and_max_leaders(latest_config->x_proto,
3616 new_max_leaders)) {
3617
2/4
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
 6 G_WARNING(
3618 "The max number of leaders was not reconfigured "
3619 " because some of the group's members do not support "
3620 "reconfiguring the max number of leaders to %" PRIu32,
3621 new_max_leaders);
3622 6 return TRUE;
3623 } else {
3624 158 return FALSE;
3625 }
3626 }
3627
3628 194 static bool_t is_unsafe_set_leaders_reconfiguration(app_data_ptr a
3629 [[maybe_unused]]) {
3630
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 194 times.
 194 assert(a->body.c_t == set_leaders_type);
3631 194 const site_def *latest_config = get_site_def();
3632
2/2
✓ Branch 0 taken 27 times.
✓ Branch 1 taken 167 times.
 194 if (incompatible_proto_and_leaders(latest_config->x_proto)) {
3633
2/4
✓ Branch 0 taken 27 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 27 times.
✗ Branch 3 not taken.
 27 G_WARNING(
3634 "The set of leaders was not reconfigured "
3635 " because some of the group's members do not support "
3636 "reconfiguring leaders");
3637 27 return TRUE;
3638 } else {
3639 167 return FALSE;
3640 }
3641 }
3642
3643 179 static bool_t is_unsafe_leaders_reconfiguration(app_data_ptr a) {
3644
2/2
✓ Branch 0 taken 340 times.
✓ Branch 1 taken 149 times.
 489 while (a) {
3645
2/3
✓ Branch 0 taken 161 times.
✓ Branch 1 taken 179 times.
✗ Branch 2 not taken.
 340 switch (a->body.c_t) {
3646 161 case set_max_leaders:
3647
2/2
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 149 times.
 161 if (is_unsafe_max_leaders_reconfiguration(a)) return TRUE;
3648 149 break;
3649 179 case set_leaders_type:
3650
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 161 times.
 179 if (is_unsafe_set_leaders_reconfiguration(a)) return TRUE;
3651 161 break;
3652 ✗ default:
3653 ✗ break;
3654 }
3655 310 a = a->next;
3656 }
3657 149 return FALSE;
3658 }
3659
3660 28 static bool_t are_there_dead_nodes_in_new_config(app_data_ptr a) {
3661
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 assert(a->body.c_t == force_config_type);
3662
3663 {
3664 28 u_int nr_nodes_to_add = a->body.app_u_u.nodes.node_list_len;
3665 28 node_address *nodes_to_change = a->body.app_u_u.nodes.node_list_val;
3666 uint32_t i;
3667
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 G_DEBUG("Checking for dead nodes in Forced Configuration")
3668
2/2
✓ Branch 0 taken 31 times.
✓ Branch 1 taken 27 times.
 58 for (i = 0; i < nr_nodes_to_add; i++) {
3669 31 node_no node = find_nodeno(get_site_def(), nodes_to_change[i].address);
3670
3671
2/2
✓ Branch 0 taken 27 times.
✓ Branch 1 taken 4 times.
 31 if (node == get_nodeno(get_site_def()))
3672 27 continue; /* No need to validate myself */
3673
3674
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
 4 if (node == VOID_NODE_NO) {
3675 ✗ G_ERROR(
3676 "%s is not in the current configuration."
3677 "Only members in the current configuration can be present"
3678 " in a forced configuration list",
3679 nodes_to_change[i].address)
3680 ✗ return TRUE;
3681 }
3682
3683
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 3 times.
 4 if (may_be_dead(get_site_def()->detected, node, task_now())) {
3684
2/4
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
 1 G_ERROR(
3685 "%s is suspected to be failed."
3686 "Only alive members in the current configuration should be "
3687 "present"
3688 " in a forced configuration list",
3689 nodes_to_change[i].address)
3690 1 return TRUE;
3691 }
3692 }
3693 }
3694
3695 27 return FALSE;
3696 }
3697
3698 /**
3699 * Reconfigure the event horizon.
3700 *
3701 * It is possible that concurrent reconfigurations take effect between the
3702 * time this reconfiguration was proposed and now.
3703 *
3704 * Particularly, it is possible that any of the concurrent reconfigurations
3705 * added a new member which does not support reconfiguring the event
3706 * horizon.
3707 *
3708 * We account for these situations by validating if the event horizon
3709 * reconfiguration is still possible under the current state.
3710 *
3711 * If it is not, this reconfiguration does not produce any effect, i.e. no
3712 * new configuration is installed.
3713 */
3714 19 bool_t handle_event_horizon(app_data_ptr a) {
3715
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 if (is_unsafe_event_horizon_reconfiguration(a)) return FALSE;
3716
3717 {
3718 19 xcom_event_horizon new_event_horizon = a->body.app_u_u.event_horizon;
3719 19 const site_def *latest_config = get_site_def();
3720 19 site_def *new_config = clone_site_def(latest_config);
3721 IFDBG(D_NONE, FN; NDBG(new_event_horizon, u));
3722 IFDBG(D_NONE, FN; NDBG(new_event_horizon, u));
3723 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("a->app_key"));
3724 add_synode_event(a->app_key););
3725
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 assert(get_site_def());
3726
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 assert(new_config);
3727 19 new_config->event_horizon = new_event_horizon;
3728 19 set_start_and_boot(new_config, a);
3729 19 site_install_action(new_config, a->body.c_t);
3730
2/4
✓ Branch 0 taken 19 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 G_INFO("The event horizon was reconfigured to %" PRIu32, new_event_horizon);
3731 }
3732 19 return TRUE;
3733 }
3734
3735 119 static bool_t handle_max_leaders(site_def *new_config, app_data_ptr a) {
3736 IFDBG(D_BASE, FN; NUMEXP(a->body.app_u_u.max_leaders));
3737
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 119 times.
 119 assert(new_config);
3738 119 new_config->max_active_leaders = a->body.app_u_u.max_leaders;
3739 119 set_start_and_boot(new_config, a);
3740
2/4
✓ Branch 0 taken 119 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 119 times.
✗ Branch 3 not taken.
 119 G_INFO("Maximum number of leaders was reconfigured to %" PRIu32,
3741 a->body.app_u_u.max_leaders);
3742 119 return TRUE;
3743 }
3744
3745 21 bool_t handle_max_leaders(app_data_ptr a) {
3746
2/2
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 9 times.
 21 if (is_unsafe_max_leaders_reconfiguration(a)) return FALSE;
3747
3748 9 site_def *new_config = clone_site_def(get_site_def());
3749 9 handle_max_leaders(new_config, a);
3750 9 site_install_action(new_config, a->body.c_t);
3751 9 return TRUE;
3752 }
3753
3754 116 static void zero_leader_array(leader_array *l) {
3755 116 l->leader_array_len = 0;
3756 116 l->leader_array_val = nullptr;
3757 116 }
3758
3759 116 static void move_leader_array(leader_array *target, leader_array *source) {
3760 /* Deallocate leader_array from target */
3761 116 xdr_free((xdrproc_t)xdr_leader_array, (char *)target);
3762 116 *target = *source;
3763 /* Zero the source */
3764 116 zero_leader_array(source);
3765 116 }
3766
3767 116 static bool_t handle_set_leaders(site_def *new_config, app_data_ptr a) {
3768 IFDBG(D_BASE, FN; NUMEXP(a->body.app_u_u.leaders.leader_array_len);
3769 NUMEXP(new_config->max_active_leaders));
3770
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 116 times.
 116 assert(new_config);
3771 /* Steal the leaders from a */
3772 116 move_leader_array(&new_config->leaders, &a->body.app_u_u.leaders);
3773 116 set_start_and_boot(new_config, a);
3774 116 return TRUE;
3775 }
3776
3777 15 bool_t handle_set_leaders(app_data_ptr a) {
3778
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 6 times.
 15 if (is_unsafe_set_leaders_reconfiguration(a)) return FALSE;
3779
3780 6 site_def *new_config = clone_site_def(get_site_def());
3781 6 handle_set_leaders(new_config, a);
3782 6 site_install_action(new_config, a->body.c_t);
3783
3/6
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 6 times.
✗ Branch 5 not taken.
 6 G_INFO("Preferred leaders were reconfigured to leaders[0]=%s",
3784 new_config->leaders.leader_array_len > 0
3785 ? new_config->leaders.leader_array_val[0].address
3786 : "n/a");
3787 6 return TRUE;
3788 }
3789
3790 140 bool_t handle_leaders(app_data_ptr a) {
3791
2/2
✓ Branch 0 taken 30 times.
✓ Branch 1 taken 110 times.
 140 if (is_unsafe_leaders_reconfiguration(a)) return FALSE;
3792 110 site_def *new_config = clone_site_def(get_site_def());
3793 110 cargo_type operation{a->body.c_t}; // Deallocate on scope exit if failure
3794 110 bool_t retval = TRUE;
3795
3/4
✓ Branch 0 taken 220 times.
✓ Branch 1 taken 110 times.
✓ Branch 2 taken 220 times.
✗ Branch 3 not taken.
 330 while (a && retval) {
3796
2/3
✓ Branch 0 taken 110 times.
✓ Branch 1 taken 110 times.
✗ Branch 2 not taken.
 220 switch (a->body.c_t) {
3797 110 case set_max_leaders:
3798
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 110 times.
 110 if (!handle_max_leaders(new_config, a)) retval = FALSE;
3799 110 break;
3800 110 case set_leaders_type:
3801
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 110 times.
 110 if (!handle_set_leaders(new_config, a)) retval = FALSE;
3802 110 break;
3803 ✗ default:
3804 ✗ break;
3805 }
3806 220 a = a->next;
3807 }
3808
1/2
✓ Branch 0 taken 110 times.
✗ Branch 1 not taken.
 110 if (retval) {
3809 110 site_install_action(new_config, operation);
3810 } else {
3811 ✗ free_site_def(new_config);
3812 }
3813 110 return retval;
3814 }
3815
3816 2348 void terminate_and_exit() {
3817 IFDBG(D_NONE, FN;);
3818 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("terminating"));)
3819
4/4
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 2347 times.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 2347 times.
 2348 XCOM_FSM(x_fsm_terminate, int_arg(0)); /* Tell xcom to stop */
3820
4/4
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 2347 times.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 2347 times.
 2348 XCOM_FSM(x_fsm_exit, int_arg(0)); /* Tell xcom to exit */
3821
1/2
✓ Branch 0 taken 2348 times.
✗ Branch 1 not taken.
 2348 if (xcom_expel_cb) xcom_expel_cb(0);
3822 2348 }
3823
3824 2169 static inline int is_empty_site(site_def const *s) {
3825 2169 return s->nodes.node_list_len == 0;
3826 }
3827
3828 3954 site_def *handle_remove_node(app_data_ptr a) {
3829 3954 site_def const *old_site = get_site_def();
3830 3954 site_def *site = clone_site_def(old_site);
3831 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&a->body.app_u_u.nodes)));
3832 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("a->app_key"));
3833 add_synode_event(a->app_key);
3834 add_event(EVENT_DUMP_PAD, string_arg("nodeno"));
3835 add_event(EVENT_DUMP_PAD, uint_arg(get_nodeno(site))););
3836
3837 3954 remove_site_def(a->body.app_u_u.nodes.node_list_len,
3838 a->body.app_u_u.nodes.node_list_val, site);
3839 3954 set_start_and_boot(site, a);
3840
1/2
✓ Branch 0 taken 3954 times.
✗ Branch 1 not taken.
 3954 if (site->x_proto >= single_writer_support) {
3841 3954 recompute_node_sets(old_site, site);
3842 3954 recompute_timestamps(old_site->detected, &old_site->nodes, site->detected,
3843 3954 &site->nodes);
3844 }
3845 3954 site_install_action(site, a->body.c_t);
3846 3954 return site;
3847 }
3848
3849 ✗ static void log_ignored_forced_config(app_data_ptr a,
3850 char const *const caller_name) {
3851 ✗ switch (a->body.c_t) {
3852 ✗ case unified_boot_type:
3853 ✗ G_DEBUG("%s: Ignoring a forced intermediate, pending unified_boot",
3854 caller_name);
3855 ✗ break;
3856 ✗ case add_node_type:
3857 ✗ G_DEBUG("%s: Ignoring a forced intermediate, pending add_node for %s",
3858 caller_name, a->body.app_u_u.nodes.node_list_val[0].address);
3859 ✗ break;
3860 ✗ case remove_node_type:
3861 ✗ G_DEBUG("%s: Ignoring a forced intermediate, pending remove_node for %s",
3862 caller_name, a->body.app_u_u.nodes.node_list_val[0].address);
3863 ✗ break;
3864 ✗ case set_event_horizon_type:
3865 ✗ G_DEBUG(
3866 "%s: Ignoring a forced intermediate, pending set_event_horizon for "
3867 "%" PRIu32,
3868 caller_name, a->body.app_u_u.event_horizon);
3869 ✗ break;
3870 ✗ case force_config_type:
3871 ✗ G_DEBUG("%s: Ignoring a forced intermediate, pending force_config",
3872 caller_name);
3873 ✗ break;
3874 ✗ case set_max_leaders:
3875 ✗ G_DEBUG(
3876 "%s: Ignoring a forced intermediate, pending set_max_leaders for "
3877 "%" PRIu32,
3878 caller_name, a->body.app_u_u.max_leaders);
3879 ✗ break;
3880 ✗ case set_leaders_type:
3881 ✗ G_DEBUG("%s: Ignoring a forced intermediate, pending set_leaders_type",
3882 caller_name);
3883 ✗ break;
3884 ✗ case abort_trans:
3885 case app_type:
3886 case begin_trans:
3887 case convert_into_local_server_type:
3888 case disable_arbitrator:
3889 case enable_arbitrator:
3890 case exit_type:
3891 case get_event_horizon_type:
3892 case get_synode_app_data_type:
3893 case prepared_trans:
3894 case remove_reset_type:
3895 case reset_type:
3896 case set_cache_limit:
3897 case view_msg:
3898 case x_terminate_and_exit:
3899 case xcom_boot_type:
3900 case xcom_set_group:
3901 case get_leaders_type:
3902 // Meaningless for any other `cargo_type`s. Ignore.
3903 ✗ break;
3904 }
3905 }
3906
3907 6661 bool_t handle_config(app_data_ptr a, bool const forced) {
3908
6/8
✓ Branch 0 taken 5824 times.
✓ Branch 1 taken 837 times.
✓ Branch 2 taken 5824 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 5720 times.
✓ Branch 5 taken 104 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 5720 times.
 6661 assert(a->body.c_t == unified_boot_type || a->body.c_t == set_max_leaders ||
3909 a->body.c_t == set_leaders_type ||
3910 a->next == nullptr); /* Reconfiguration commands are not batched. */
3911 {
3912 6661 bool_t success = FALSE;
3913
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 6661 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 6661 times.
 6661 if (forced &&
3914 ✗ should_ignore_forced_config_or_view(get_executor_site()->x_proto)) {
3915 ✗ log_ignored_forced_config(a, "handle_config");
3916 ✗ goto end;
3917 }
3918
5/7
✓ Branch 0 taken 837 times.
✓ Branch 1 taken 1747 times.
✓ Branch 2 taken 3954 times.
✓ Branch 3 taken 19 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 104 times.
✗ Branch 6 not taken.
 6661 switch (a->body.c_t) {
3919 837 case unified_boot_type:
3920 837 success = (install_node_group(a) != nullptr);
3921
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 837 times.
 837 assert(success);
3922 837 break;
3923 1747 case add_node_type:
3924 /*
3925 * May fail if meanwhile the event horizon was reconfigured and the
3926 * node is incompatible.
3927 */
3928 1747 success = (handle_add_node(a) != nullptr);
3929 1747 break;
3930 3954 case remove_node_type:
3931 ADD_DBG(D_BASE,
3932 add_event(EVENT_DUMP_PAD, string_arg("got remove_node_type"));)
3933 3954 success = (handle_remove_node(a) != nullptr);
3934
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3954 times.
 3954 assert(success);
3935 3954 break;
3936 19 case set_event_horizon_type:
3937 /* May fail if meanwhile an incompatible node joined. */
3938 19 success = handle_event_horizon(a);
3939 19 break;
3940 ✗ case force_config_type:
3941 ✗ success = (install_node_group(a) != nullptr);
3942 ✗ assert(success);
3943 ✗ break;
3944 104 case set_max_leaders:
3945 case set_leaders_type:
3946 104 success = handle_leaders(a);
3947
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 104 times.
 104 assert(success);
3948 104 break;
3949 ✗ default:
3950 ✗ assert(FALSE); /* Boy oh boy, something is really wrong... */
3951 break;
3952 }
3953 6661 end:
3954 6661 return success;
3955 }
3956 }
3957
3958 7498 static inline int is_member(site_def const *site) {
3959 7498 return site->nodeno != VOID_NODE_NO;
3960 }
3961
3962 /*
3963 Execute xcom message stream.
3964
3965 Beware of the exit logic in this task, which is both simple and
3966 not so simple. Consider three configs C1 and C2. C1 has two
3967 nodes, A and B. C2 has only node B. C3 is empty. A config with
3968 message number N will be activated after a delay of (at least)
3969 alpha messages, where alpha is the size of the pipeline (or the
3970 event horizon).
3971
3972 So, C1.start = C1+alpha, and C2.start = C2+alpha. A, which is re‐
3973 moved from C1, cannot exit until a majority of nodes in the new
3974 config C2 (in this case B) has learned all the messages from con‐
3975 fig C1, which means all messages less than C2.start. How can A
3976 know that a majority of C2 has learned those messages?
3977
3978 If we denote the first message that is not yet decided (and exe‐
3979 cuted) by E, the proposers will not try to propose messages with
3980 number >= E+alpha, and all incoming tcp messages with message
3981 number >= E+alpha will be ignored. E is incremented by the ex‐
3982 ecutor task, so all messages < E are known. This means that when
3983 the value of E+alpha is known, all messages up to and including E
3984 are also known, although not all messages E+1..E+alpha‐1 neces‐
3985 sarily are known.
3986
3987 This leads to the requirement that a node which is removed (A)
3988 needs to wait until it knows the value of C2.start+alpha, since
3989 by then it knows that a majority of the nodes in C2 are ready to
3990 execute C2.start, which in turn implies that a majority of nodes
3991 in C2 knows all the values from config C1. Note that the last
3992 message that should be delivered to the application by a node
3993 that is leaving C1 is C2.start‐1, which is the last message of
3994 C1.
3995
3996 How does a node that is removed get to know values from the next
3997 config? There are two ways, and we use both. First, the node
3998 that tries to exit can simply ask for the message. get_xcom_mes‐
3999 sage() will do this for all messages <= max_synode, but it may
4000 take some time. Second, the nodes of C2 can send the messages
4001 C2.start..C2.start+alpha to the nodes that are removed (nodes
4002 that are in C1 but not in C2). inform_removed() does this. We
4003 take care to handle the case where configs are close enough that
4004 C0 < C1 <= C0+alpha by tracking the oldest config that contains
4005 nodes that are leaving.
4006
4007 This takes care of nodes leaving C1. What about nodes that leave
4008 C2? C3 is empty, so B, which is leaving C2, cannot wait for mes‐
4009 sages from C3. But since C3 is empty, there is no need to wait.
4010 It can exit immediately after having executed C3.start‐1, the
4011 last message of C2. What if C3.start‐1 < C2.start+alpha? This can
4012 happen if C2 and C3 are close. In that case, B will exit before A
4013 gets the chance to learn C2.start+alpha, which will leave A hang‐
4014 ing forever. Clearly, we need to impose an additional constraint,
4015 that C3.start must be greater than C2.start+alpha. This is taken
4016 care of by the special test for an empty config.
4017
4018 Complicated and confusing? Not really, but there is a clean and
4019 simple solution which has not been implemented yet, since it re‐
4020 quires more changes to the consensus logic. If we require that
4021 for the messages C2..C2.start‐1 we have a majority from both the
4022 nodes in C1 and the nodes in C2, the nodes not in C2 can exit
4023 when they have executed message C2.start‐1, since we then know
4024 that a majority of the nodes of C2 has agreed on those messages
4025 as well, so they do not depend on the nodes not in C2 any more.
4026 This holds even if C2 is empty. Note that requiring a majority
4027 from both C1 and C2 is different from requiring a majority from
4028 C1+C2, which means that the proposer logic needs to consider an‐
4029 swers from two different sets of acceptors for those messages.
4030 Since acceptors are identified by their node number, and the node
4031 numbers need not be the same for both configs, we need to main‐
4032 tain a mapping between the nodes numbers of any two consecutive
4033 configs. Alternatively, we could remove the node numbers alto‐
4034 gether, and always use a unique, unchanging ID for a node, like
4035 IP address + port.
4036
4037 TODO:
4038
4039 Move the delayed delivery logic into MCM-specific code, since it is
4040 only needed by MCM. Is it still needed?
4041
4042 Rewrite exit logic as FSM with more states. (RUN, EMPTY_EXIT,
4043 NOT_MEMBER_EXIT) to avoid unnecessary tests.
4044
4045 */
4046
4047 /* FIFO which tracks the message numbers where we should deliver queued messages
4048 or
4049 inform the removed nodes */
4050 #define FIFO_SIZE 1000
4051 static struct {
4052 int n;
4053 int front;
4054 int rear;
4055 synode_no q[FIFO_SIZE];
4056 } delay_fifo;
4057
4058 13322 static inline int addone(int i) { return ((i + 1) % FIFO_SIZE); }
4059
4060 /* Is queue empty? */
4061 860187 static inline int fifo_empty() { return delay_fifo.n <= 0; }
4062
4063 /* Is queue full? */
4064 6661 static inline int fifo_full() { return delay_fifo.n >= FIFO_SIZE; }
4065
4066 /* Insert in queue */
4067 6661 static inline void fifo_insert(synode_no s) {
4068
1/2
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
 6661 if (!fifo_full()) {
4069 6661 delay_fifo.n++;
4070 6661 delay_fifo.q[delay_fifo.rear] = s;
4071 6661 delay_fifo.rear = addone(delay_fifo.rear);
4072 }
4073 6661 }
4074
4075 /* Extract first from queue */
4076 6661 static inline synode_no fifo_extract() {
4077
1/2
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
 6661 if (!fifo_empty()) {
4078 6661 synode_no ret = delay_fifo.q[delay_fifo.front];
4079 6661 delay_fifo.front = addone(delay_fifo.front);
4080 6661 delay_fifo.n--;
4081 6661 return ret;
4082 } else {
4083 ✗ return null_synode;
4084 }
4085 }
4086
4087 /* Return first in queue, but do not dequeue */
4088 244252 static inline synode_no fifo_front() {
4089
1/2
✓ Branch 0 taken 244252 times.
✗ Branch 1 not taken.
 244252 if (!fifo_empty()) {
4090 244252 return delay_fifo.q[delay_fifo.front];
4091 } else {
4092 ✗ return null_synode;
4093 }
4094 }
4095
4096 struct execute_context;
4097 typedef struct execute_context execute_context;
4098
4099 typedef void (*exec_fp)(execute_context *xc);
4100
4101 struct execute_context {
4102 pax_machine *p;
4103 int n;
4104 int old_n;
4105 double old_t;
4106 synode_no exit_synode;
4107 synode_no delivery_limit;
4108 exec_fp state;
4109 int exit_flag; /* To avoid state explosion */
4110 int inform_index;
4111 };
4112
4113 static void dump_exec_state(execute_context *xc [[maybe_unused]],
4114 long dbg [[maybe_unused]]);
4115 static int x_check_exit(execute_context *xc);
4116 static int x_check_execute_inform(execute_context *xc);
4117 static void x_fetch(execute_context *xc);
4118 static void x_execute(execute_context *xc);
4119 static void x_check_increment_fetch(execute_context *xc);
4120 static void x_check_increment_execute(execute_context *xc);
4121 static void x_terminate(execute_context *xc);
4122
4123 struct fp_name {
4124 exec_fp fp;
4125 char const *name;
4126 };
4127
4128 #define NAME(f) \
4129 { f, #f }
4130
4131 /* List of fp, name pairs */
4132 static struct fp_name MY_ATTRIBUTE((unused)) oblist[] = {
4133 NAME(x_fetch), NAME(x_execute), NAME(x_terminate), {nullptr, nullptr}};
4134 #undef NAME
4135
4136 #if TASK_DBUG_ON
4137 /* purecov: begin deadcode */
4138 char const *get_fp_name(exec_fp fp) {
4139 struct fp_name *list = oblist;
4140 while (list->fp) {
4141 if (list->fp == fp) return list->name;
4142 list++;
4143 }
4144 return "no such fp";
4145 }
4146 /* purecov: end */
4147 #endif
4148
4149 6661 static void setup_exit_handling(execute_context *xc, site_def *site) {
4150 synode_no delay_until;
4151
2/2
✓ Branch 0 taken 4492 times.
✓ Branch 1 taken 2169 times.
 6661 if (is_member(site)) {
4152 4492 delay_until = compute_delay(site->start, site->event_horizon);
4153 } else { /* Not in this site */
4154 /* See if site will be empty when we leave. If the new site
4155 * is empty, we should exit after having delivered the last
4156 * message from the old site. */
4157
4158 /* Note limit of delivery. We should never deliver anything after the
4159 * start of the next site. */
4160 2169 xc->delivery_limit = site->start;
4161
4162 /* If we are not a member of the new site, we should exit
4163 after having seen enough messages beyond the end of the current site.
4164 This ensures that a majority of the next site will have agreed upon all
4165 messages that belong to the current site.
4166 */
4167 2169 xc->exit_synode = compute_delay(site->start, site->event_horizon);
4168
2/2
✓ Branch 0 taken 873 times.
✓ Branch 1 taken 1296 times.
 2169 if (is_empty_site(site)) {
4169 /* If site is empty, increase start to allow nodes to terminate before
4170 * start. This works as if there was a non-empty group after the
4171 * exit_synode, effectively allowing the majority of the current group
4172 * to agree on all messages up to exit_synode.
4173 */
4174 873 site->start = compute_delay(
4175 compute_delay(site->start, site->event_horizon), site->event_horizon);
4176 }
4177
3/4
✓ Branch 0 taken 2169 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2139 times.
✓ Branch 3 taken 30 times.
 2169 if (!synode_lt(xc->exit_synode, max_synode)) {
4178 /* We need messages from the next site, so set max_synode accordingly.
4179 */
4180
2/4
✓ Branch 0 taken 2139 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2139 times.
✗ Branch 3 not taken.
 2139 set_max_synode(incr_synode(xc->exit_synode));
4181 }
4182 /* Note where we switch to execute and inform removed nodes */
4183 2169 delay_until = xc->exit_synode;
4184
4185 IFDBG(D_EXEC, FN; SYCEXP(delay_until); SYCEXP(executed_msg);
4186 SYCEXP(max_synode));
4187 IFDBG(D_EXEC, FN; SYCEXP(xc->exit_synode); SYCEXP(executed_msg);
4188 SYCEXP(max_synode));
4189
4190 /* Note that we will exit */
4191 2169 xc->exit_flag = 1;
4192 }
4193
4194 /* Ensure that max_synode is greater than trigger for delivery
4195 */
4196
3/4
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4470 times.
✓ Branch 3 taken 2191 times.
 6661 if (synode_gt(delay_until, max_synode))
4197
2/4
✓ Branch 0 taken 4470 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4470 times.
✗ Branch 3 not taken.
 4470 set_max_synode(incr_msgno(delay_until));
4198 6661 fifo_insert(delay_until);
4199 6661 (xc->inform_index)++;
4200
4201 /* If I am the leader, will propose no-ops until current max_synode
4202 */
4203 6661 }
4204
4205 /* Called immediately after we have got a new message.
4206 Terminate if we have no site.
4207 Otherwise, handle config messages immediately.
4208 Afterwards, switch to check_exit_fetch. */
4209 592701 static void x_fetch(execute_context *xc) {
4210 /* Execute unified_boot immediately, but do not deliver site message
4211 * until we are ready to execute messages from the new site
4212 * definition. At that point we can be certain that a majority have
4213 * learned everything from the old site. */
4214
4215 592701 app_data *app = xc->p->learner.msg->a;
4216
6/6
✓ Branch 0 taken 213443 times.
✓ Branch 1 taken 379258 times.
✓ Branch 2 taken 6691 times.
✓ Branch 3 taken 206752 times.
✓ Branch 4 taken 6661 times.
✓ Branch 5 taken 586040 times.
 599392 if (app && is_config(app->body.c_t) &&
4217
2/2
✓ Branch 0 taken 6661 times.
✓ Branch 1 taken 30 times.
 6691 synode_gt(executed_msg, get_site_def()->boot_key)) /* Redo test */
4218 {
4219 6661 site_def *site = nullptr;
4220 bool_t reconfiguration_successful =
4221 6661 handle_config(app, (xc->p->learner.msg->force_delivery != 0));
4222
1/2
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
 6661 if (reconfiguration_successful) {
4223 /* If the reconfiguration failed then it does not have any
4224 * effect. What follows only makes sense if the reconfiguration
4225 * took effect. */
4226
1/2
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
 6661 set_last_received_config(executed_msg);
4227
1/2
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
 6661 synode_no min_synode = min_proposer_synode();
4228
4/6
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6661 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
✓ Branch 5 taken 6660 times.
 13322 if (synode_eq(null_synode, min_synode) ||
4229
3/4
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 6660 times.
 6661 synode_lt(delivered_msg, min_synode))
4230
1/2
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
 1 min_synode = get_last_delivered_msg();
4231
1/2
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
 6661 garbage_collect_site_defs(min_synode);
4232
1/2
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
 6661 site = get_site_def_rw();
4233
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6661 times.
 6661 if (site == nullptr) {
4234 ✗ xc->state = x_terminate;
4235 ✗ return;
4236 }
4237 IFDBG(D_EXEC, FN; STRLIT("new config "); SYCEXP(site->boot_key););
4238
4239
1/2
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
 6661 if (xc->exit_flag == 0) {
4240 /* We have not yet set the exit trigger */
4241
1/2
✓ Branch 0 taken 6661 times.
✗ Branch 1 not taken.
 6661 setup_exit_handling(xc, site);
4242 }
4243 }
4244 } else {
4245 IFDBG(D_EXEC, FN; SYCEXP(executed_msg); SYCEXP(get_site_def()->boot_key));
4246 }
4247 /* Check for exit and increment executed_msg */
4248 592701 x_check_increment_fetch(xc);
4249 }
4250
4251 /* Push messages to nodes that have been removed.
4252 Signal switch to execute when nothing left to push by returning 1 */
4253 595952 static int x_check_execute_inform(execute_context *xc) {
4254 IFDBG(D_EXEC, FN; SYCEXP(fifo_front()); SYCEXP(executed_msg);
4255 SYCEXP(xc->exit_synode); NDBG(xc->exit_flag, d));
4256
2/2
✓ Branch 0 taken 358423 times.
✓ Branch 1 taken 237529 times.
 595952 if (fifo_empty()) {
4257 358423 return 1;
4258
2/2
✓ Branch 0 taken 6661 times.
✓ Branch 1 taken 230868 times.
 237529 } else if (!synode_lt(executed_msg, fifo_front())) {
4259 6661 while (
4260
4/4
✓ Branch 0 taken 6723 times.
✓ Branch 1 taken 6599 times.
✓ Branch 2 taken 6661 times.
✓ Branch 3 taken 6661 times.
 20045 !fifo_empty() &&
4261
2/2
✓ Branch 0 taken 6661 times.
✓ Branch 1 taken 62 times.
 6723 !synode_lt(executed_msg, fifo_front())) { /* More than one may match */
4262 6661 inform_removed(xc->inform_index, 0);
4263 6661 fifo_extract();
4264 6661 (xc->inform_index)--;
4265 }
4266 6661 garbage_collect_servers();
4267 6661 return 1;
4268 }
4269 230868 dump_exec_state(xc, D_EXEC);
4270 230868 return 0;
4271 }
4272
4273 /* Check for exit and return 1 if we should exit. */
4274 1167415 static int x_check_exit(execute_context *xc) {
4275 /* See if we should exit when having seen this message */
4276
4/4
✓ Branch 0 taken 114926 times.
✓ Branch 1 taken 1052489 times.
✓ Branch 2 taken 44892 times.
✓ Branch 3 taken 70034 times.
 1212307 return (xc->exit_flag && !synode_lt(executed_msg, xc->exit_synode) &&
4277
2/2
✓ Branch 0 taken 2169 times.
✓ Branch 1 taken 42723 times.
 1212307 !synode_lt(delivered_msg, xc->delivery_limit));
4278 }
4279
4280 /* Terminate if we should exit, else increment executed_msg and see if we
4281 * should switch to execute */
4282 595952 static void x_check_increment_fetch(execute_context *xc) {
4283
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 595952 times.
 595952 if (x_check_exit(xc)) {
4284 ✗ xc->state = x_terminate;
4285 } else {
4286 595952 SET_EXECUTED_MSG(incr_synode(executed_msg));
4287
2/2
✓ Branch 0 taken 365084 times.
✓ Branch 1 taken 230868 times.
 595952 if (x_check_execute_inform(xc)) {
4288 365084 xc->state = x_execute;
4289 }
4290 }
4291 595952 }
4292
4293 /* Terminate if we should exit, else increment delivered_msg and see if we
4294 * should switch to fetch */
4295 571463 static void x_check_increment_execute(execute_context *xc) {
4296
2/2
✓ Branch 0 taken 2169 times.
✓ Branch 1 taken 569294 times.
 571463 if (x_check_exit(xc)) {
4297 2169 xc->state = x_terminate;
4298 } else {
4299 /* Increment delivered_msg and switch to fetch if delivered_msg equals
4300 * executed_msg; */
4301 569294 delivered_msg = incr_synode(delivered_msg);
4302
2/2
✓ Branch 0 taken 362915 times.
✓ Branch 1 taken 206379 times.
 569294 if (synode_eq(delivered_msg, executed_msg)) {
4303 362915 xc->state = x_fetch;
4304 }
4305 }
4306 571463 }
4307
4308 /* Deliver one message if it should be delivered. Switch state to see if
4309 we should exit */
4310 571463 static void x_execute(execute_context *xc) {
4311 571463 site_def *x_site = find_site_def_rw(delivered_msg);
4312
4313 IFDBG(D_EXEC, FN; SYCEXP(delivered_msg); SYCEXP(delivered_msg);
4314 SYCEXP(executed_msg); SYCEXP(xc->exit_synode); NDBG(xc->exit_flag, d));
4315 571463 if (!is_cached(delivered_msg)) {
4316 /* purecov: begin deadcode */
4317 #ifdef TASK_EVENT_TRACE
4318 dump_task_events();
4319 #endif
4320 /* purecov: end */
4321 }
4322
2/2
✓ Branch 0 taken 568232 times.
✓ Branch 1 taken 3231 times.
 571463 if (!ignore_message(delivered_msg, x_site, "x_execute")) {
4323
1/2
✓ Branch 0 taken 568232 times.
✗ Branch 1 not taken.
 568232 assert(is_cached(delivered_msg) && "delivered_msg should have been cached");
4324 568232 xc->p = get_cache(delivered_msg);
4325
2/2
✓ Branch 0 taken 213391 times.
✓ Branch 1 taken 354841 times.
 568232 if ((xc->p)->learner.msg->msg_type != no_op) {
4326 /* Avoid delivery after start if we should exit */
4327
6/6
✓ Branch 0 taken 2239 times.
✓ Branch 1 taken 211152 times.
✓ Branch 2 taken 2236 times.
✓ Branch 3 taken 3 times.
✓ Branch 4 taken 213388 times.
✓ Branch 5 taken 3 times.
 213391 if (xc->exit_flag == 0 || synode_lt(delivered_msg, xc->delivery_limit)) {
4328 IFDBG(D_EXEC, FN; STRLIT("executing "); SYCEXP(delivered_msg);
4329 SYCEXP(executed_msg); SYCEXP(xc->delivery_limit);
4330 NDBG(xc->exit_flag, d));
4331 213388 last_delivered_msg = delivered_msg;
4332 213388 execute_msg(find_site_def_rw(delivered_msg), xc->p, xc->p->learner.msg);
4333 }
4334 }
4335 }
4336 /* Garbage collect old servers */
4337
2/2
✓ Branch 0 taken 7190 times.
✓ Branch 1 taken 564273 times.
 571463 if (synode_eq(delivered_msg, x_site->start)) {
4338 7190 garbage_collect_servers();
4339 }
4340 #if defined(TASK_DBUG_ON) && TASK_DBUG_ON
4341 IFDBG(D_EXEC, perf_dbg(&xc->n, &xc->old_n, &xc->old_t));
4342 #endif
4343 /* Check for exit and increment delivered_msg */
4344 571463 x_check_increment_execute(xc);
4345 571463 }
4346
4347 static execute_context *debug_xc;
4348
4349 1348899 static void dump_exec_state(execute_context *xc [[maybe_unused]],
4350 long dbg [[maybe_unused]]) {
4351 IFDBG(dbg, FN; SYCEXP(executed_msg); SYCEXP(delivered_msg);
4352 SYCEXP(max_synode); SYCEXP(last_delivered_msg); NDBG(delay_fifo.n, d);
4353 NDBG(delay_fifo.front, d); NDBG(delay_fifo.rear, d);
4354 SYCEXP(fifo_front()); SYCEXP(xc->exit_synode);
4355 SYCEXP(xc->delivery_limit); NDBG(xc->exit_flag, d);
4356 NDBG(xc->inform_index, d); NDBG(prop_started, d);
4357 NDBG(prop_finished, d););
4358 1348899 }
4359
4360 1111503 static void dump_debug_exec_state() {
4361
1/2
✓ Branch 0 taken 1111503 times.
✗ Branch 1 not taken.
 1111503 if (debug_xc) dump_exec_state(debug_xc, D_EXEC);
4362 1111503 }
4363
4364 /* Terminate the excutor_task. */
4365 2169 static void x_terminate(execute_context *xc) {
4366 2169 dump_exec_state(xc, D_BUG);
4367 2169 xc->state = nullptr;
4368 2169 }
4369
4370 520971 static int executor_task(task_arg arg [[maybe_unused]]) {
4371 DECL_ENV
4372 execute_context xc;
4373 2197 ENV_INIT
4374 2197 END_ENV_INIT
4375 END_ENV;
4376 /* xcom_debug_mask = D_BUG; */
4377 IFDBG(D_EXEC, FN; NDBG(stack->sp->state, d); SYCEXP(executed_msg););
4378
6/10
✓ Branch 0 taken 2197 times.
✓ Branch 1 taken 516605 times.
✓ Branch 2 taken 2169 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 2197 times.
✓ Branch 6 taken 2197 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 2197 times.
 520971 TASK_BEGIN
4379 2197 ep->xc.p = nullptr;
4380 2197 ep->xc.n = 0;
4381 2197 ep->xc.old_n = 0;
4382 2197 ep->xc.old_t = task_now();
4383 2197 ep->xc.exit_synode = null_synode;
4384 2197 ep->xc.delivery_limit = null_synode;
4385 2197 ep->xc.exit_flag = 0;
4386 2197 ep->xc.inform_index = -1;
4387 2197 delay_fifo.n = 0;
4388 2197 delay_fifo.front = 0;
4389 2197 delay_fifo.rear = 0;
4390 2197 debug_xc = &ep->xc;
4391
4392
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2197 times.
 2197 if (executed_msg.msgno == 0) executed_msg.msgno = 1;
4393 2197 delivered_msg = executed_msg;
4394 2197 ep->xc.state = x_fetch;
4395 2197 executor_site = find_site_def_rw(executed_msg);
4396
4397 /* The following loop implements a state machine based on function pointers,
4398 effectively acting as non-local gotos.
4399 The functions all operate on data in the execution context xc, and
4400 switch state by setting xc->state to the function corresponding to the
4401 new state.
4402 */
4403
3/4
✓ Branch 0 taken 1171781 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1169612 times.
✓ Branch 3 taken 2169 times.
 1171781 while (!xcom_shutdown && ep->xc.state != nullptr) {
4404 IFDBG(D_EXEC, FN; STRLIT(get_fp_name(ep->xc.state)););
4405
2/2
✓ Branch 0 taken 595980 times.
✓ Branch 1 taken 573632 times.
 1169612 if (ep->xc.state == x_fetch) { /* Special case because of task macros */
4406
2/2
✓ Branch 0 taken 3251 times.
✓ Branch 1 taken 592729 times.
 595980 if (ignore_message(executed_msg, executor_site, "executor_task")) {
4407 IFDBG(D_EXEC, FN; STRLIT("ignoring message "); SYCEXP(executed_msg));
4408 3251 x_check_increment_fetch(&ep->xc); /* Just increment past losers */
4409 } else {
4410 IFDBG(D_EXEC, FN; STRLIT("fetching message "); SYCEXP(executed_msg));
4411
10/14
✓ Branch 0 taken 592722 times.
✓ Branch 1 taken 516612 times.
✓ Branch 2 taken 21 times.
✓ Branch 3 taken 592701 times.
✓ Branch 4 taken 516612 times.
✓ Branch 5 taken 592701 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 516605 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 516605 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 516605 times.
✓ Branch 13 taken 592701 times.
 1625939 TASK_CALL(get_xcom_message(&ep->xc.p, executed_msg, FIND_MAX));
4412 IFDBG(D_EXEC, FN; STRLIT("got message "); SYCEXP(ep->xc.p->synode);
4413 COPY_AND_FREE_GOUT(dbg_app_data(ep->xc.p->learner.msg->a)));
4414 592701 x_fetch(&ep->xc);
4415 }
4416 } else {
4417 573632 ep->xc.state(&ep->xc);
4418 }
4419 }
4420
4421 /* Inform all removed nodes before we exit */
4422 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("terminating"));)
4423 2169 inform_removed(ep->xc.inform_index, 1);
4424 2169 dump_exec_state(&ep->xc, D_EXEC);
4425
4426 #ifndef NO_DELAYED_TERMINATION
4427 IFDBG(D_EXEC, FN; STRLIT("delayed terminate and exit"));
4428
4429 /* Wait to allow messages to propagate */
4430
3/6
✗ Branch 0 not taken.
✓ Branch 1 taken 2169 times.
✓ Branch 2 taken 2169 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 2169 times.
 4338 TASK_DELAY(TERMINATE_DELAY);
4431
4432 /* Start termination of xcom */
4433 2169 terminate_and_exit();
4434 #endif
4435
4436 2190 FINALLY
4437 2190 dump_exec_state(&ep->xc, D_EXEC);
4438 IFDBG(D_BUG, FN; STRLIT(" shutdown "); SYCEXP(executed_msg);
4439 NDBG(task_now(), f));
4440
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2190 times.
 2190 TASK_END;
4441 }
4442
4443 9325 static synode_no get_sweep_start() {
4444 9325 synode_no find = executed_msg;
4445 9325 find.node = get_nodeno(find_site_def(find));
4446
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9325 times.
 9325 if (find.node < executed_msg.node) {
4447 ✗ find = incr_msgno(find);
4448 }
4449 9325 return find;
4450 }
4451
4452 /* Allow takeover of channel if not all are leaders. We may need to adjust
4453 * this if we allow any subset of the nodes as leaders */
4454 159259 static bool allow_channel_takeover(site_def const *site) {
4455 159259 return site->max_active_leaders != active_leaders_all;
4456 }
4457
4458 159259 static void broadcast_noop(synode_no find, pax_machine *p) {
4459 159259 site_def const *site = find_site_def(find);
4460
4461 /*
4462 If we allow channel hijacking, we cannot send skip_op, but need consensus.
4463 There are two options here:
4464
4465 a) We unconditionally propose a `no_op` using the regular 3-phase Paxos
4466 protocol, or
4467 b) We propose a `no_op` using the 2-phase Paxos protocol *if* we
4468 are sure that no other Proposer will try to run the 2-phase Paxos
4469 protocol on `find`. If we are not sure, we propose using the 3-phase Paxos
4470 protocol.
4471
4472 Option (a) is always safe, but we pay the cost of 3-phase Paxos.
4473 Option (b) can be implemented by having the leaders keep track of the
4474 synods they allocate to non-leaders. If we are the leader for `find` and we
4475 allocated it to a non-leader, we must use 3-phase Paxos here to be safe
4476 against the non-leader using 2-phase Paxos. If we never allocated `find` to
4477 a non-leader, we can use 2-phase Paxos here if we ensure we don't allocate
4478 `find` to a non-leader afterwards.
4479
4480 We go with option (a) because there is no evidence that the additional
4481 complexity that option (b) requires is worthwhile.
4482 */
4483
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 159259 times.
 159259 if (allow_channel_takeover(site)) {
4484 ✗ propose_noop(find, p); // Single leader
4485 } else {
4486 159259 skip_msg(pax_msg_new(find, site)); // Multiple leaders
4487 }
4488 159259 }
4489
4490 843579 static int sweeper_task(task_arg arg [[maybe_unused]]) {
4491 DECL_ENV
4492 synode_no find;
4493 2197 ENV_INIT
4494 2197 END_ENV_INIT
4495 END_ENV;
4496
4497
6/10
✓ Branch 0 taken 2197 times.
✓ Branch 1 taken 427586 times.
✓ Branch 2 taken 413796 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 2197 times.
✓ Branch 6 taken 2197 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 2197 times.
 843579 TASK_BEGIN
4498
4499 2197 ep->find = get_sweep_start();
4500
4501
1/2
✓ Branch 0 taken 841389 times.
✗ Branch 1 not taken.
 841389 while (!xcom_shutdown) {
4502 841389 ep->find.group_id =
4503 841389 executed_msg.group_id; /* In case group id has changed */
4504 #ifndef AGGRESSIVE_SWEEP
4505 while (!is_only_task()) {
4506 TASK_YIELD;
4507 }
4508 #endif
4509 ADD_DBG(D_NONE, add_event(EVENT_DUMP_PAD, string_arg("sweeper ready"));
4510 add_synode_event(executed_msg););
4511 /* IFDBG(D_NONE, FN; STRLIT("ready to run "); */
4512 /* SYCEXP(executed_msg); SYCEXP(max_synode);
4513 * SYCEXP(ep->find));
4514 */
4515
6/6
✓ Branch 0 taken 684266 times.
✓ Branch 1 taken 426860 times.
✓ Branch 2 taken 457896 times.
✓ Branch 3 taken 226370 times.
✓ Branch 4 taken 457896 times.
✓ Branch 5 taken 653230 times.
 1111126 while (synode_lt(ep->find, max_synode) && !too_far(ep->find)) {
4516 /* pax_machine * pm = hash_get(ep->find); */
4517 457896 pax_machine *pm = nullptr;
4518 ADD_DBG(D_NONE,
4519 add_event(EVENT_DUMP_PAD, string_arg("sweeper examining"));
4520 add_synode_event(ep->find););
4521
2/2
✓ Branch 0 taken 188159 times.
✓ Branch 1 taken 269737 times.
 457896 if (ep->find.node == VOID_NODE_NO) {
4522
2/2
✓ Branch 0 taken 7128 times.
✓ Branch 1 taken 181031 times.
 188159 if (synode_gt(executed_msg, ep->find)) {
4523 7128 ep->find = get_sweep_start();
4524 }
4525
1/2
✓ Branch 0 taken 188159 times.
✗ Branch 1 not taken.
 188159 if (ep->find.node == VOID_NODE_NO) goto deactivate;
4526 }
4527 269737 pm = get_cache(ep->find);
4528 ADD_DBG(D_CONS, add_event(EVENT_DUMP_PAD, string_arg("sweeper checking"));
4529 add_synode_event(ep->find);
4530 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(pm->op)));
4531 add_event(EVENT_DUMP_PAD, string_arg("pm"));
4532 add_event(EVENT_DUMP_PAD, void_arg(pm)););
4533
3/4
✓ Branch 0 taken 269737 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 269051 times.
✓ Branch 3 taken 686 times.
 269737 if (pm && !pm->force_delivery) { /* We want full 3 phase Paxos for
4534 forced messages */
4535 ADD_DBG(
4536 D_CONS, add_event(EVENT_DUMP_PAD, string_arg("sweeper checking"));
4537 add_synode_event(ep->find);
4538 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(pm->op)));
4539 add_event(EVENT_DUMP_PAD, string_arg("is_busy_machine"));
4540 add_event(EVENT_DUMP_PAD, int_arg(is_busy_machine(pm)));
4541 add_event(EVENT_DUMP_PAD, string_arg("pm->acceptor.promise.cnt"));
4542 add_event(EVENT_DUMP_PAD, int_arg(pm->acceptor.promise.cnt));
4543 add_event(EVENT_DUMP_PAD, string_arg("finished(pm)"));
4544 add_event(EVENT_DUMP_PAD, int_arg(finished(pm)));
4545 add_event(EVENT_DUMP_PAD, string_arg("pm->acceptor.msg"));
4546 add_event(EVENT_DUMP_PAD, void_arg(pm->acceptor.msg)););
4547 /* IFDBG(D_NONE, FN; dbg_pax_machine(pm)); */
4548
2/2
✓ Branch 0 taken 264782 times.
✓ Branch 1 taken 1698 times.
 535531 if (!is_busy_machine(pm) && pm->acceptor.promise.cnt == 0 &&
4549
7/8
✓ Branch 0 taken 266480 times.
✓ Branch 1 taken 2571 times.
✓ Branch 2 taken 160244 times.
✓ Branch 3 taken 104538 times.
✓ Branch 4 taken 160244 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 160244 times.
✓ Branch 7 taken 108807 times.
 535531 !pm->acceptor.msg && !finished(pm)) {
4550 ADD_DBG(
4551 D_CONS, add_event(EVENT_DUMP_PAD, string_arg("sweeper skipping"));
4552 add_synode_event(ep->find);
4553 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(pm->op))););
4554 160244 site_def *config = find_site_def_rw(ep->find);
4555 // Do not send noop if single writer, since there normally will be
4556 // no holes in the message sequence, and it may interfere with
4557 // messages delegated to secondary nodes.
4558
4/4
✓ Branch 0 taken 159259 times.
✓ Branch 1 taken 985 times.
✓ Branch 2 taken 159259 times.
✓ Branch 3 taken 985 times.
 319503 if (config->max_active_leaders != 1 &&
4559
1/2
✓ Branch 0 taken 159259 times.
✗ Branch 1 not taken.
 159259 !ignore_message(ep->find, config, "sweeper_task")) {
4560 159259 broadcast_noop(ep->find, pm);
4561 }
4562 IFDBG(D_NONE, FN; STRLIT("skipping "); SYCEXP(ep->find));
4563 }
4564 }
4565 269737 ep->find = incr_msgno(ep->find);
4566 }
4567 653230 deactivate:
4568
2/2
✓ Branch 0 taken 427593 times.
✓ Branch 1 taken 413796 times.
 841389 if (!synode_lt(ep->find, max_synode)) {
4569
4/6
✗ Branch 0 not taken.
✓ Branch 1 taken 427586 times.
✓ Branch 2 taken 427586 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1626 times.
✓ Branch 5 taken 425960 times.
 855179 TASK_DEACTIVATE;
4570 } else {
4571
4/6
✗ Branch 0 not taken.
✓ Branch 1 taken 413796 times.
✓ Branch 2 taken 413796 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 564 times.
✓ Branch 5 taken 413232 times.
 827592 TASK_DELAY(0.010); /* Let poll_wait check for IO */
4572 }
4573 }
4574 ✗ FINALLY
4575 IFDBG(D_BUG, FN; STRLIT(" shutdown sweeper "); SYCEXP(executed_msg);
4576 NDBG(task_now(), f));
4577
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2190 times.
 2190 TASK_END;
4578 }
4579
4580 632707 static double wakeup_delay(double old) {
4581 632707 double const minimum_threshold = 0.1;
4582 #ifdef EXECUTOR_TASK_AGGRESSIVE_NO_OP
4583 double const maximum_threshold = 1.0;
4584 #else
4585 632707 double const maximum_threshold = 20.0;
4586 #endif /* EXECUTOR_TASK_AGGRESSIVE_NO_OP */
4587 632707 double retval = 0.0;
4588
2/2
✓ Branch 0 taken 349964 times.
✓ Branch 1 taken 282743 times.
 632707 if (0.0 == old) {
4589 349964 double m = median_time();
4590 349964 double const fuzz = 5.0;
4591 IFDBG(D_BUG, FN; NDBG(m, f));
4592 // Guard against unreasonable estimates of median consensus time
4593
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 349964 times.
 349964 if (m <= 0.0) m = minimum_threshold;
4594
2/2
✓ Branch 0 taken 927 times.
✓ Branch 1 taken 349037 times.
 349964 if (m > maximum_threshold / fuzz) m = (maximum_threshold / fuzz) / 2.0;
4595 349964 retval = minimum_threshold + fuzz * m + m * xcom_drand48();
4596 } else {
4597 282743 retval = old * 1.4142136; /* Exponential backoff */
4598 }
4599 /* If we exceed maximum, choose a random value in the max/2..max interval */
4600
2/2
✓ Branch 0 taken 258 times.
✓ Branch 1 taken 632449 times.
 632707 if (retval > maximum_threshold) {
4601 258 double const low = maximum_threshold / 2.0;
4602 258 retval = low + xcom_drand48() * (maximum_threshold - low);
4603 }
4604 IFDBG(D_BUG, FN; NDBG(retval, f));
4605 632707 return retval;
4606 }
4607
4608 21774 static site_def const *init_noop(synode_no find, pax_machine *p) {
4609 /* Prepare to send a noop */
4610 21774 site_def const *site = find_site_def(find);
4611 IFDBG(D_NONE, FN; SYCEXP(find); SYCEXP(executed_msg));
4612
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 21774 times.
 21774 assert(!too_far(find));
4613 21774 replace_pax_msg(&p->proposer.msg, pax_msg_new(find, site));
4614
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 21774 times.
 21774 assert(p->proposer.msg);
4615 21774 create_noop(p->proposer.msg);
4616 21774 return site;
4617 }
4618
4619 21774 static void propose_noop(synode_no find, pax_machine *p) {
4620 21774 site_def const *site = init_noop(find, p);
4621 21774 pax_msg *clone = clone_pax_msg(p->proposer.msg);
4622
1/2
✓ Branch 0 taken 21774 times.
✗ Branch 1 not taken.
 21774 if (clone != nullptr) {
4623 IFDBG(D_CONS, FN; SYCEXP(find));
4624 21774 push_msg_3p(site, p, clone, find, no_op);
4625 } else {
4626 /* purecov: begin inspected */
4627 ✗ G_DEBUG("Unable to propose NoOp due to an OOM error.");
4628 /* purecov: end */
4629 }
4630 21774 }
4631
4632 #if 0
4633 static void propose_noop_2p(synode_no find, pax_machine *p) {
4634 site_def const *site = init_noop(find, p);
4635 IFDBG(D_CONS, FN; SYCEXP(find));
4636 push_msg_2p(site, p);
4637 }
4638 #endif
4639
4640 426200 static void send_read(synode_no find) {
4641 /* Prepare to send a read_op */
4642 426200 site_def const *site = find_site_def(find);
4643
4644 IFDBG(D_NONE, FN; NDBG(get_maxnodes(site), u); NDBG(get_nodeno(site), u););
4645 ADD_DBG(D_CONS, add_event(EVENT_DUMP_PAD, string_arg("find"));
4646 add_synode_event(find); add_event(EVENT_DUMP_PAD, string_arg("site"));
4647 add_event(EVENT_DUMP_PAD, void_arg((void *)find_site_def_rw(find)));
4648 add_event(EVENT_DUMP_PAD, string_arg("get_nodeno(site)"));
4649 add_event(EVENT_DUMP_PAD, uint_arg(get_nodeno(site))););
4650
4651 /* See if node number matches ours */
4652
2/2
✓ Branch 0 taken 425940 times.
✓ Branch 1 taken 260 times.
 426200 if (site) {
4653
2/2
✓ Branch 0 taken 268774 times.
✓ Branch 1 taken 157166 times.
 425940 if (find.node != get_nodeno(site)) {
4654
1/2
✓ Branch 0 taken 268774 times.
✗ Branch 1 not taken.
 268774 pax_msg *pm = pax_msg_new(find, site);
4655
1/2
✓ Branch 0 taken 268774 times.
✗ Branch 1 not taken.
 268774 ref_msg(pm);
4656
1/2
✓ Branch 0 taken 268774 times.
✗ Branch 1 not taken.
 268774 create_read(site, pm);
4657 IFDBG(D_NONE, FN; SYCEXP(find););
4658
4659 IFDBG(D_NONE, FN; NDBG(get_maxnodes(site), u); NDBG(get_nodeno(site), u);
4660 PTREXP(pm));
4661 /* send_server_msg(site, find.node, pm); */
4662 #if 0
4663 send_to_others(site, pm, "send_read");
4664 #else
4665 /* If we have no node number, ask all the others */
4666
3/4
✓ Branch 0 taken 268774 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 84425 times.
✓ Branch 3 taken 184349 times.
 268774 if (get_nodeno(site) == VOID_NODE_NO)
4667
1/2
✓ Branch 0 taken 84425 times.
✗ Branch 1 not taken.
 84425 send_to_others(site, pm, "send_read");
4668 else
4669 /* Ask a random node */
4670
1/2
✓ Branch 0 taken 184349 times.
✗ Branch 1 not taken.
 184349 send_to_someone(site, pm, "send_read");
4671 #endif
4672
1/2
✓ Branch 0 taken 268774 times.
✗ Branch 1 not taken.
 268774 unref_msg(&pm);
4673 } else { /* If node number matches our own number, ask all the others */
4674
1/2
✓ Branch 0 taken 157166 times.
✗ Branch 1 not taken.
 157166 pax_msg *pm = pax_msg_new(find, site);
4675
1/2
✓ Branch 0 taken 157166 times.
✗ Branch 1 not taken.
 157166 ref_msg(pm);
4676
1/2
✓ Branch 0 taken 157166 times.
✗ Branch 1 not taken.
 157166 create_read(site, pm);
4677
1/2
✓ Branch 0 taken 157166 times.
✗ Branch 1 not taken.
 157166 send_to_others(site, pm, "send_read");
4678
1/2
✓ Branch 0 taken 157166 times.
✗ Branch 1 not taken.
 157166 unref_msg(&pm);
4679 }
4680 }
4681 426200 }
4682
4683 /* Find missing values */
4684
4685 67919 static int ok_to_propose(pax_machine *p) {
4686
6/6
✓ Branch 0 taken 61139 times.
✓ Branch 1 taken 6780 times.
✓ Branch 2 taken 28074 times.
✓ Branch 3 taken 33065 times.
✓ Branch 4 taken 22889 times.
✓ Branch 5 taken 11965 times.
 90808 int retval = (is_forcing_node(p) || !recently_active(p)) && !finished(p) &&
4687
2/2
✓ Branch 0 taken 21774 times.
✓ Branch 1 taken 1115 times.
 22889 !is_busy_machine(p);
4688 IFDBG(D_NONE, FN; NDBG(p->synode.node, u); NDBG(recently_active(p), d);
4689 NDBG(finished(p), d); NDBG(is_busy_machine(p), d); NDBG(retval, d));
4690 67919 return retval;
4691 }
4692
4693 387557 static void read_missing_values(int n) {
4694 387557 synode_no find = executed_msg;
4695 387557 synode_no end = max_synode;
4696 387557 int i = 0;
4697
4698 IFDBG(D_NONE, FN; SYCEXP(find); SYCEXP(end));
4699
5/6
✓ Branch 0 taken 387557 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 54117 times.
✓ Branch 3 taken 333440 times.
✓ Branch 4 taken 333440 times.
✓ Branch 5 taken 54117 times.
 441674 if (synode_gt(executed_msg, max_synode) ||
4700
2/4
✓ Branch 0 taken 54117 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 54117 times.
 54117 synode_eq(executed_msg, null_synode))
4701 333440 return;
4702
4703
9/12
✓ Branch 0 taken 680731 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 649388 times.
✓ Branch 3 taken 31343 times.
✓ Branch 4 taken 649388 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 649388 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 626614 times.
✓ Branch 9 taken 22774 times.
✓ Branch 10 taken 626614 times.
✓ Branch 11 taken 54117 times.
 680731 while (!synode_gt(find, end) && i < n && !too_far(find)) {
4704
1/2
✓ Branch 0 taken 626614 times.
✗ Branch 1 not taken.
 626614 pax_machine *p = force_get_cache(find);
4705 ADD_DBG(D_NONE, add_synode_event(find); add_synode_event(end);
4706 add_event(EVENT_DUMP_PAD, string_arg("active "));
4707 add_event(EVENT_DUMP_PAD, int_arg(recently_active(p)));
4708 add_event(EVENT_DUMP_PAD, string_arg("finished "));
4709 add_event(EVENT_DUMP_PAD, int_arg(finished(p)));
4710 add_event(EVENT_DUMP_PAD, string_arg("busy "));
4711 add_event(EVENT_DUMP_PAD, int_arg(is_busy_machine(p))););
4712 IFDBG(D_NONE, FN; SYCEXP(find); SYCEXP(end); NDBG(recently_active(p), d);
4713 NDBG(finished(p), d); NDBG(is_busy_machine(p), d));
4714
10/12
✓ Branch 0 taken 626614 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 430289 times.
✓ Branch 3 taken 196325 times.
✓ Branch 4 taken 427210 times.
✓ Branch 5 taken 3079 times.
✓ Branch 6 taken 427210 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 425932 times.
✓ Branch 9 taken 1278 times.
✓ Branch 10 taken 425932 times.
✓ Branch 11 taken 200682 times.
 626614 if (!recently_active(p) && !finished(p) && !is_busy_machine(p)) {
4715
1/2
✓ Branch 0 taken 425932 times.
✗ Branch 1 not taken.
 425932 send_read(find);
4716 }
4717
1/2
✓ Branch 0 taken 626614 times.
✗ Branch 1 not taken.
 626614 find = incr_synode(find);
4718 626614 i++;
4719 }
4720 }
4721
4722 129055 static void propose_missing_values(int n) {
4723 129055 synode_no find = executed_msg;
4724 129055 synode_no end = max_synode;
4725 129055 int i = 0;
4726
4727 IFDBG(D_NONE, FN; NDBG(get_maxnodes(get_site_def()), u); SYCEXP(find);
4728 SYCEXP(end));
4729
5/6
✓ Branch 0 taken 129055 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6645 times.
✓ Branch 3 taken 122410 times.
✓ Branch 4 taken 122410 times.
✓ Branch 5 taken 6645 times.
 135700 if (synode_gt(executed_msg, max_synode) ||
4730
2/4
✓ Branch 0 taken 6645 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 6645 times.
 6645 synode_eq(executed_msg, null_synode))
4731 122410 return;
4732
4733 IFDBG(D_NONE, FN; SYCEXP(find); SYCEXP(end));
4734 6645 i = 0;
4735
9/12
✓ Branch 0 taken 74926 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 70958 times.
✓ Branch 3 taken 3968 times.
✓ Branch 4 taken 70958 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 70958 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 68283 times.
✓ Branch 9 taken 2675 times.
✓ Branch 10 taken 68283 times.
✓ Branch 11 taken 6643 times.
 74926 while (!synode_gt(find, end) && i < n && !too_far(find)) {
4736
1/2
✓ Branch 0 taken 68283 times.
✗ Branch 1 not taken.
 68283 pax_machine *p = force_get_cache(find);
4737
2/2
✓ Branch 0 taken 3546 times.
✓ Branch 1 taken 64737 times.
 68283 if (wait_forced_config) {
4738 3546 force_pax_machine(p, 1);
4739 }
4740 IFDBG(D_NONE, FN; NDBG(ok_to_propose(p), d); TIMECEXP(task_now());
4741 TIMECEXP(p->last_modified); SYCEXP(find));
4742
1/2
✓ Branch 0 taken 68283 times.
✗ Branch 1 not taken.
 68283 site_def *site = find_site_def_rw(find);
4743
3/4
✓ Branch 0 taken 68283 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✓ Branch 3 taken 68281 times.
 68283 if (get_nodeno(site) == VOID_NODE_NO) break;
4744
5/6
✓ Branch 0 taken 68281 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 67919 times.
✓ Branch 3 taken 362 times.
✓ Branch 4 taken 21774 times.
✓ Branch 5 taken 46507 times.
 136200 if (!ignore_message(find, site, "propose_missing_values") &&
4745
3/4
✓ Branch 0 taken 67919 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 21774 times.
✓ Branch 3 taken 46145 times.
 67919 ok_to_propose(p)) {
4746
1/2
✓ Branch 0 taken 21774 times.
✗ Branch 1 not taken.
 21774 propose_noop(find, p);
4747 }
4748
1/2
✓ Branch 0 taken 68281 times.
✗ Branch 1 not taken.
 68281 find = incr_synode(find);
4749 68281 i++;
4750 }
4751 }
4752
4753 /* Message handlers */
4754
4755 /*
4756 Reply to the sender of a message.
4757 Avoid using the outbound TCP connection to the node that sent the message, since
4758 it is simpler and safer to always use the same TCP connection as the one the
4759 message arrived on. We then know that the answer will always go to the same
4760 client (and the same instance of that client) that sent the request.
4761 */
4762 #define reply_msg(m) \
4763 { \
4764 if (is_local_node((m)->from, site)) { \
4765 dispatch_op(site, m, NULL); \
4766 } else { \
4767 link_into(&(msg_link_new((m), (m)->from)->l), reply_queue); \
4768 } \
4769 }
4770
4771 #define CREATE_REPLY(x) \
4772 pax_msg *reply = NULL; \
4773 CLONE_PAX_MSG(reply, x)
4774
4775 #define SEND_REPLY \
4776 reply_msg(reply); \
4777 replace_pax_msg(&reply, NULL)
4778
4779 329091 bool_t safe_app_data_copy(pax_msg **target, app_data_ptr source) {
4780 329091 copy_app_data(&(*target)->a, source);
4781
3/4
✓ Branch 0 taken 211233 times.
✓ Branch 1 taken 117858 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 211233 times.
 329091 if ((*target)->a == nullptr && source != nullptr) {
4782 ✗ oom_abort = 1;
4783 ✗ replace_pax_msg(target, nullptr);
4784 ✗ return FALSE;
4785 }
4786 329091 return TRUE;
4787 }
4788
4789 118783 static pax_msg *create_learn_msg_for_ignorant_node(pax_machine *p, pax_msg *pm,
4790 synode_no synode) {
4791
2/4
✓ Branch 0 taken 118783 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 118783 times.
✗ Branch 3 not taken.
 118783 CREATE_REPLY(pm);
4792 IFDBG(D_NONE, FN; SYCEXP(synode));
4793 118783 reply->synode = synode;
4794 118783 reply->proposal = p->learner.msg->proposal;
4795 118783 reply->msg_type = p->learner.msg->msg_type;
4796
1/2
✓ Branch 0 taken 118783 times.
✗ Branch 1 not taken.
 118783 safe_app_data_copy(&reply, p->learner.msg->a);
4797
1/2
✓ Branch 0 taken 118783 times.
✗ Branch 1 not taken.
 118783 if (reply != nullptr) set_learn_type(reply);
4798 /* set_unique_id(reply, p->learner.msg->unique_id); */
4799 118783 return reply;
4800 }
4801
4802 111280 static void teach_ignorant_node(site_def const *site, pax_machine *p,
4803 pax_msg *pm, synode_no synode,
4804 linkage *reply_queue) {
4805
1/2
✓ Branch 0 taken 111280 times.
✗ Branch 1 not taken.
 111280 pax_msg *reply = create_learn_msg_for_ignorant_node(p, pm, synode);
4806
5/12
✓ Branch 0 taken 111280 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 111280 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 111280 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 111280 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 111280 times.
✗ Branch 11 not taken.
 111280 if (reply != nullptr) SEND_REPLY;
4807 111280 }
4808
4809 /* Handle incoming read */
4810 333399 static void handle_read(site_def const *site, pax_machine *p,
4811 linkage *reply_queue, pax_msg *pm) {
4812 IFDBG(D_NONE, FN; BALCEXP(pm->proposal); BALCEXP(p->acceptor.promise);
4813 if (p->acceptor.msg) BALCEXP(p->acceptor.msg->proposal);
4814 STRLIT("type "); STRLIT(pax_msg_type_to_str(pm->msg_type)));
4815
4816
2/2
✓ Branch 0 taken 111280 times.
✓ Branch 1 taken 222119 times.
 333399 if (finished(p)) { /* We have learned a value */
4817 111280 teach_ignorant_node(site, p, pm, pm->synode, reply_queue);
4818 }
4819 333399 }
4820
4821 36334 static pax_msg *create_ack_prepare_msg(pax_machine *p, pax_msg *pm,
4822 synode_no synode) {
4823
2/4
✓ Branch 0 taken 36334 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 36334 times.
✗ Branch 3 not taken.
 36334 CREATE_REPLY(pm);
4824 36334 reply->synode = synode;
4825
2/2
✓ Branch 0 taken 3969 times.
✓ Branch 1 taken 32365 times.
 36334 if (accepted(p)) { /* We have accepted a value */
4826 3969 reply->proposal = p->acceptor.msg->proposal;
4827 3969 reply->msg_type = p->acceptor.msg->msg_type;
4828 IFDBG(D_NONE, FN; STRLIT(" already accepted value "); SYCEXP(synode));
4829 3969 reply->op = ack_prepare_op;
4830
1/2
✓ Branch 0 taken 3969 times.
✗ Branch 1 not taken.
 3969 safe_app_data_copy(&reply, p->acceptor.msg->a);
4831 } else {
4832 IFDBG(D_NONE, FN; STRLIT(" no value synode "); SYCEXP(synode));
4833 32365 reply->op = ack_prepare_empty_op;
4834 }
4835 36334 return reply;
4836 }
4837
4838 39397 pax_msg *handle_simple_prepare(pax_machine *p, pax_msg *pm, synode_no synode) {
4839 39397 pax_msg *reply = nullptr;
4840
2/2
✓ Branch 0 taken 3030 times.
✓ Branch 1 taken 36367 times.
 39397 if (finished(p)) { /* We have learned a value */
4841 IFDBG(D_NONE, FN; SYCEXP(synode); BALCEXP(pm->proposal);
4842 NDBG(finished(p), d));
4843 3030 reply = create_learn_msg_for_ignorant_node(p, pm, synode);
4844 } else {
4845 int greater =
4846 36367 gt_ballot(pm->proposal,
4847 p->acceptor.promise); /* Paxos acceptor phase 1 decision */
4848 IFDBG(D_NONE, FN; SYCEXP(synode); BALCEXP(pm->proposal); NDBG(greater, d));
4849
5/6
✓ Branch 0 taken 33 times.
✓ Branch 1 taken 36334 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 33 times.
✓ Branch 4 taken 36334 times.
✓ Branch 5 taken 33 times.
 36367 if (greater || noop_match(p, pm)) {
4850 36334 p->last_modified = task_now();
4851
1/2
✓ Branch 0 taken 36334 times.
✗ Branch 1 not taken.
 36334 if (greater) {
4852 36334 p->acceptor.promise = pm->proposal; /* promise to not accept any less */
4853 }
4854 36334 reply = create_ack_prepare_msg(p, pm, synode);
4855 }
4856 }
4857 39397 return reply;
4858 }
4859
4860 /* Handle incoming prepare */
4861 39388 static void handle_prepare(site_def const *site, pax_machine *p,
4862 linkage *reply_queue, pax_msg *pm) {
4863 ADD_DBG(D_CONS, add_synode_event(p->synode);
4864 add_event(EVENT_DUMP_PAD, string_arg("pm->from"));
4865 add_event(EVENT_DUMP_PAD, uint_arg(pm->from));
4866 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(pm->op)));
4867 add_event(EVENT_DUMP_PAD, string_arg("proposal"));
4868 add_ballot_event(pm->proposal);
4869 add_event(EVENT_DUMP_PAD, string_arg("promise"));
4870 add_ballot_event(p->acceptor.promise););
4871 IFDBG(D_NONE, FN; BALCEXP(pm->proposal); BALCEXP(p->acceptor.promise);
4872 if (p->acceptor.msg) BALCEXP(p->acceptor.msg->proposal);
4873 STRLIT("type "); STRLIT(pax_msg_type_to_str(pm->msg_type)));
4874
4875 {
4876
1/2
✓ Branch 0 taken 39388 times.
✗ Branch 1 not taken.
 39388 pax_msg *reply = handle_simple_prepare(p, pm, pm->synode);
4877
8/12
✓ Branch 0 taken 39355 times.
✓ Branch 1 taken 33 times.
✓ Branch 2 taken 39355 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 25372 times.
✓ Branch 5 taken 13983 times.
✓ Branch 6 taken 25372 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 13983 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 39388 times.
✗ Branch 11 not taken.
 39388 if (reply != nullptr) SEND_REPLY;
4878 }
4879 39388 }
4880
4881 61554 bool_t check_propose(site_def const *site, pax_machine *p) {
4882 IFDBG(D_NONE, FN; SYCEXP(p->synode);
4883 COPY_AND_FREE_GOUT(dbg_machine_nodeset(p, get_maxnodes(site))););
4884 PAX_MSG_SANITY_CHECK(p->proposer.msg);
4885 {
4886 61554 bool_t can_propose = FALSE;
4887
2/2
✓ Branch 0 taken 20695 times.
✓ Branch 1 taken 40859 times.
 61554 if (prep_majority(site, p)) {
4888 20695 p->proposer.msg->proposal = p->proposer.bal;
4889 20695 BIT_ZERO(p->proposer.prop_nodeset);
4890 20695 p->proposer.msg->synode = p->synode;
4891 20695 init_propose_msg(p->proposer.msg);
4892 20695 p->proposer.sent_prop = p->proposer.bal;
4893 20695 can_propose = TRUE;
4894 }
4895 61554 return can_propose;
4896 }
4897 }
4898
4899 424161 static bool learn_ok(site_def const *site, pax_machine const *p) {
4900
3/4
✓ Branch 0 taken 424161 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 236288 times.
✓ Branch 3 taken 187873 times.
 424161 return get_nodeno(site) != VOID_NODE_NO && prop_majority(site, p);
4901 }
4902
4903 212316 static pax_msg *check_learn(site_def const *site, pax_machine *p) {
4904 IFDBG(D_NONE, FN; SYCEXP(p->synode);
4905 COPY_AND_FREE_GOUT(dbg_machine_nodeset(p, get_maxnodes(site))););
4906 PAX_MSG_SANITY_CHECK(p->proposer.msg);
4907 {
4908 212316 pax_msg *learn_msg = nullptr;
4909
2/2
✓ Branch 0 taken 118274 times.
✓ Branch 1 taken 94042 times.
 212316 if (learn_ok(site, p)) {
4910 118274 p->proposer.msg->synode = p->synode;
4911
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 118274 times.
 118274 if (p->proposer.msg->receivers) free_bit_set(p->proposer.msg->receivers);
4912 118274 p->proposer.msg->receivers = clone_bit_set(p->proposer.prep_nodeset);
4913 118274 BIT_SET(get_nodeno(site), p->proposer.msg->receivers);
4914 if (no_duplicate_payload) {
4915 118274 learn_msg = create_tiny_learn_msg(p, p->proposer.msg);
4916 } else {
4917 /* purecov: begin deadcode */
4918 init_learn_msg(p->proposer.msg);
4919 learn_msg = p->proposer.msg;
4920 /* purecov: end */
4921 }
4922 118274 p->proposer.sent_learn = p->proposer.bal;
4923 }
4924 212316 return learn_msg;
4925 }
4926 }
4927
4928 599450 static void do_learn(site_def const *site [[maybe_unused]], pax_machine *p,
4929 pax_msg *m) {
4930 ADD_DBG(D_CONS, add_synode_event(p->synode);
4931 add_event(EVENT_DUMP_PAD, string_arg("m->from"));
4932 add_event(EVENT_DUMP_PAD, uint_arg(m->from));
4933 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(m->op)));
4934 add_event(EVENT_DUMP_PAD, string_arg("proposal"));
4935 add_ballot_event(m->proposal);
4936 add_event(EVENT_DUMP_PAD, string_arg("promise"));
4937 add_ballot_event(p->acceptor.promise););
4938 /* FN; SYCEXP(p->synode); SYCEXP(m->synode); STRLIT(NEWLINE); */
4939 IFDBG(D_NONE, FN; SYCEXP(p->synode); SYCEXP(m->synode);
4940 dbg_bitset(m->receivers, get_maxnodes(site)););
4941
2/2
✓ Branch 0 taken 217419 times.
✓ Branch 1 taken 382031 times.
 599450 if (m->a) m->a->chosen = TRUE;
4942 599450 replace_pax_msg(&p->acceptor.msg, m);
4943 599450 replace_pax_msg(&p->learner.msg, m);
4944 /*
4945 Track memory used by client data in the cache.
4946 If we do not care about instances that are being decided,
4947 it is only necessary to compute the added memory when we
4948 record the outcome of a consensus round.
4949 */
4950 599450 add_cache_size(p);
4951 /* Shrink the cache size if necessary */
4952 599450 shrink_cache();
4953 599450 }
4954
4955 35102 bool_t handle_simple_ack_prepare(site_def const *site, pax_machine *p,
4956 pax_msg *m) {
4957
1/2
✓ Branch 0 taken 35102 times.
✗ Branch 1 not taken.
 35102 if (get_nodeno(site) != VOID_NODE_NO)
4958 35102 BIT_SET(m->from, p->proposer.prep_nodeset);
4959
4960 {
4961 35102 bool_t can_propose = FALSE;
4962
4/4
✓ Branch 0 taken 3848 times.
✓ Branch 1 taken 31254 times.
✓ Branch 2 taken 28 times.
✓ Branch 3 taken 35074 times.
 38950 if (m->op == ack_prepare_op &&
4963
2/2
✓ Branch 0 taken 28 times.
✓ Branch 1 taken 3820 times.
 3848 gt_ballot(m->proposal, p->proposer.msg->proposal)) { /* greater */
4964 28 replace_pax_msg(&p->proposer.msg, m);
4965
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 assert(p->proposer.msg);
4966 }
4967
2/2
✓ Branch 0 taken 34952 times.
✓ Branch 1 taken 150 times.
 35102 if (gt_ballot(m->reply_to, p->proposer.sent_prop)) {
4968 34952 can_propose = check_propose(site, p);
4969 }
4970 35102 return can_propose;
4971 }
4972 }
4973
4974 /* Other node has already accepted a value */
4975 36251 static void handle_ack_prepare(site_def const *site, pax_machine *p,
4976 pax_msg *m) {
4977 ADD_DBG(D_CONS, add_synode_event(p->synode);
4978 add_event(EVENT_DUMP_PAD, string_arg("m->from"));
4979 add_event(EVENT_DUMP_PAD, uint_arg(m->from));
4980 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(m->op))););
4981
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 36251 times.
 36251 assert(m);
4982 IFDBG(D_NONE, FN; if (p->proposer.msg) BALCEXP(p->proposer.msg->proposal);
4983 BALCEXP(p->proposer.bal); BALCEXP(m->reply_to);
4984 BALCEXP(p->proposer.sent_prop); SYCEXP(m->synode));
4985 /*
4986 If the node is preparing a Noop for another node's slot, it is possible
4987 that the leader of the slot has since proposed a value. Hence, there is
4988 no need to move forward if we know that the value has been accepted. This
4989 also prevents changing the size of a learned pax_machine, which would
4990 cause inconsistent reporting of memory usage in P_S.
4991 */
4992
2/2
✓ Branch 0 taken 1082 times.
✓ Branch 1 taken 35169 times.
 36251 if (finished(p)) return;
4993
4994
3/4
✓ Branch 0 taken 35169 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 35093 times.
✓ Branch 3 taken 76 times.
 70338 if (m->from != VOID_NODE_NO &&
4995
2/2
✓ Branch 0 taken 35093 times.
✓ Branch 1 taken 76 times.
 35169 eq_ballot(p->proposer.bal, m->reply_to)) { /* answer to my prepare */
4996 35093 bool_t can_propose = handle_simple_ack_prepare(site, p, m);
4997
2/2
✓ Branch 0 taken 10460 times.
✓ Branch 1 taken 24633 times.
 35093 if (can_propose) send_propose_msg(p->proposer.msg);
4998 }
4999 }
5000
5001 /* #define AUTO_MSG(p,synode) {if(!(p)){replace_pax_msg(&(p),
5002 * pax_msg_new(synode, site));} */
5003
5004 234532 static pax_msg *create_ack_accept_msg(pax_msg *m, synode_no synode) {
5005
2/4
✓ Branch 0 taken 234532 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 234532 times.
✗ Branch 3 not taken.
 234532 CREATE_REPLY(m);
5006 234532 reply->op = ack_accept_op;
5007 234532 reply->synode = synode;
5008 234532 return reply;
5009 }
5010
5011 239005 pax_msg *handle_simple_accept(pax_machine *p, pax_msg *m, synode_no synode) {
5012 239005 pax_msg *reply = nullptr;
5013
2/2
✓ Branch 0 taken 4473 times.
✓ Branch 1 taken 234532 times.
 239005 if (finished(p)) { /* We have learned a value */
5014 4473 reply = create_learn_msg_for_ignorant_node(p, m, synode);
5015 234532 } else if (!gt_ballot(p->acceptor.promise,
5016
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 234532 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 234532 times.
✗ Branch 5 not taken.
 234532 m->proposal) || /* Paxos acceptor phase 2 decision */
5017 ✗ noop_match(p, m)) {
5018 IFDBG(D_NONE, FN; SYCEXP(m->synode); STRLIT("accept ");
5019 BALCEXP(m->proposal));
5020 234532 p->last_modified = task_now();
5021 234532 replace_pax_msg(&p->acceptor.msg, m);
5022 234532 reply = create_ack_accept_msg(m, synode);
5023 }
5024 239005 return reply;
5025 }
5026
5027 /* Accecpt value if promise is not greater */
5028 238990 static void handle_accept(site_def const *site, pax_machine *p,
5029 linkage *reply_queue, pax_msg *m) {
5030 IFDBG(D_NONE, FN; BALCEXP(p->acceptor.promise); BALCEXP(m->proposal);
5031 STREXP(pax_msg_type_to_str(m->msg_type)));
5032 PAX_MSG_SANITY_CHECK(m);
5033 ADD_DBG(D_CONS, add_synode_event(p->synode);
5034 add_event(EVENT_DUMP_PAD, string_arg("m->from"));
5035 add_event(EVENT_DUMP_PAD, uint_arg(m->from));
5036 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(m->op)));
5037 add_event(EVENT_DUMP_PAD, string_arg("proposal"));
5038 add_ballot_event(m->proposal);
5039 add_event(EVENT_DUMP_PAD, string_arg("promise"));
5040 add_ballot_event(p->acceptor.promise););
5041
5042 {
5043
1/2
✓ Branch 0 taken 238990 times.
✗ Branch 1 not taken.
 238990 pax_msg *reply = handle_simple_accept(p, m, m->synode);
5044
1/2
✓ Branch 0 taken 238990 times.
✗ Branch 1 not taken.
 238990 if (reply != nullptr) {
5045
6/10
✓ Branch 0 taken 238990 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 120284 times.
✓ Branch 3 taken 118706 times.
✓ Branch 4 taken 120284 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 118706 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 238990 times.
✗ Branch 9 not taken.
 238990 SEND_REPLY;
5046 IFDBG(D_CONS, FN; STRLIT("activating sweeper on accept of ");
5047 SYCEXP(m->synode));
5048
1/2
✓ Branch 0 taken 238990 times.
✗ Branch 1 not taken.
 238990 activate_sweeper();
5049 }
5050 }
5051 238990 }
5052
5053 /* Handle answer to accept */
5054 234846 pax_msg *handle_simple_ack_accept(site_def const *site, pax_machine *p,
5055 pax_msg *m) {
5056 234846 pax_msg *learn_msg = nullptr;
5057
4/6
✓ Branch 0 taken 234846 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 234846 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 234831 times.
✓ Branch 5 taken 15 times.
 469692 if (get_nodeno(site) != VOID_NODE_NO && m->from != VOID_NODE_NO &&
5058
2/2
✓ Branch 0 taken 234831 times.
✓ Branch 1 taken 15 times.
 234846 eq_ballot(p->proposer.bal, m->reply_to)) { /* answer to my accept */
5059 234831 BIT_SET(m->from, p->proposer.prop_nodeset);
5060
2/2
✓ Branch 0 taken 212316 times.
✓ Branch 1 taken 22515 times.
 234831 if (gt_ballot(m->proposal, p->proposer.sent_learn)) {
5061 212316 learn_msg = check_learn(site, p);
5062 }
5063 }
5064 234846 return learn_msg;
5065 }
5066 234831 static void handle_ack_accept(site_def const *site, pax_machine *p,
5067 pax_msg *m) {
5068 ADD_DBG(D_CONS, add_synode_event(p->synode);
5069 add_event(EVENT_DUMP_PAD, string_arg("m->from"));
5070 add_event(EVENT_DUMP_PAD, uint_arg(m->from));
5071 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(m->op))););
5072 IFDBG(D_NONE, FN; SYCEXP(m->synode); BALCEXP(p->proposer.bal);
5073 BALCEXP(p->proposer.sent_learn); BALCEXP(m->proposal);
5074 BALCEXP(m->reply_to););
5075 IFDBG(D_NONE, FN; SYCEXP(p->synode);
5076 if (p->acceptor.msg) BALCEXP(p->acceptor.msg->proposal);
5077 BALCEXP(p->proposer.bal); BALCEXP(m->reply_to););
5078
5079 {
5080 234831 pax_msg *learn_msg = handle_simple_ack_accept(site, p, m);
5081
2/2
✓ Branch 0 taken 118268 times.
✓ Branch 1 taken 116563 times.
 234831 if (learn_msg != nullptr) {
5082
1/2
✓ Branch 0 taken 118268 times.
✗ Branch 1 not taken.
 118268 if (learn_msg->op == tiny_learn_op) {
5083 118268 send_tiny_learn_msg(site, learn_msg);
5084 } else {
5085 /* purecov: begin deadcode */
5086 ✗ assert(learn_msg->op == learn_op);
5087 ✗ send_learn_msg(site, learn_msg);
5088 /* purecov: end */
5089 }
5090 }
5091 }
5092 234831 }
5093
5094 /* Handle incoming learn. */
5095 static void activate_sweeper();
5096 215634 void handle_tiny_learn(site_def const *site, pax_machine *pm, pax_msg *p) {
5097
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 215634 times.
 215634 assert(p->msg_type != no_op);
5098
2/2
✓ Branch 0 taken 215369 times.
✓ Branch 1 taken 265 times.
 215634 if (pm->acceptor.msg) {
5099 /* BALCEXP(pm->acceptor.msg->proposal); */
5100
2/2
✓ Branch 0 taken 215366 times.
✓ Branch 1 taken 3 times.
 215369 if (eq_ballot(pm->acceptor.msg->proposal, p->proposal)) {
5101 215366 pm->acceptor.msg->op = learn_op;
5102 215366 pm->last_modified = task_now();
5103 215366 update_max_synode(p);
5104 215366 paxos_fsm(pm, site, paxos_learn, p);
5105 215366 handle_learn(site, pm, pm->acceptor.msg);
5106 } else {
5107 3 send_read(p->synode);
5108 IFDBG(D_NONE, FN; STRLIT("tiny_learn"); SYCEXP(p->synode);
5109 BALCEXP(pm->acceptor.msg->proposal); BALCEXP(p->proposal));
5110 }
5111 } else {
5112 265 send_read(p->synode);
5113 IFDBG(D_NONE, FN; STRLIT("tiny_learn"); SYCEXP(p->synode);
5114 BALCEXP(p->proposal));
5115 }
5116 215634 }
5117
5118 6429 static void force_pax_machine(pax_machine *p, int enforcer) {
5119
2/2
✓ Branch 0 taken 1763 times.
✓ Branch 1 taken 4666 times.
 6429 if (!p->enforcer) { /* Not if already marked as forcing node */
5120
2/2
✓ Branch 0 taken 1501 times.
✓ Branch 1 taken 262 times.
 1763 if (enforcer) { /* Only if forcing node */
5121 /* Increase ballot count with a large increment without overflowing */
5122 /* p->proposer.bal.cnt may be -1. */
5123 1501 int32_t delta = (INT32_MAX - MAX(p->proposer.bal.cnt, 0)) / 3;
5124 1501 p->proposer.bal.cnt += delta;
5125 }
5126 }
5127 6429 p->force_delivery = 1;
5128 6429 p->enforcer = enforcer;
5129 6429 }
5130
5131 /* Configure all messages in interval start, end to be forced */
5132 57 static void force_interval(synode_no start, synode_no end, int enforcer) {
5133
2/2
✓ Branch 0 taken 2346 times.
✓ Branch 1 taken 57 times.
 2403 while (!synode_gt(start, end)) {
5134 2346 pax_machine *p = get_cache(start);
5135
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2346 times.
 2346 if (get_nodeno(find_site_def(start)) == VOID_NODE_NO) break;
5136
5137 /* The forcing node will call force_interval twice, first when
5138 the new config is originally installed, and again when it
5139 receives it as an xcom message. start may be the same, but
5140 end will be greater the second time, since it is calculated
5141 based on the message number of the incoming config. Since the forcing
5142 node is the one responsible for delivering all messages until the
5143 start of the new site, it is important that all instances belonging to
5144 the old site are correctly marked. */
5145
5146
2/2
✓ Branch 0 taken 593 times.
✓ Branch 1 taken 1753 times.
 2346 if (p->enforcer) enforcer = 1; /* Extend to new instances */
5147 2346 force_pax_machine(p, enforcer);
5148
5149 /* Old nodesets are null and void */
5150 2346 BIT_ZERO(p->proposer.prep_nodeset);
5151 2346 BIT_ZERO(p->proposer.prop_nodeset);
5152 2346 start = incr_synode(start);
5153 }
5154 57 }
5155
5156 57 static void start_force_config(site_def *s, int enforcer) {
5157
1/2
✓ Branch 0 taken 57 times.
✗ Branch 1 not taken.
 57 synode_no end = add_event_horizon(s->boot_key);
5158
5159 IFDBG(D_NONE, FN; SYCEXP(executed_msg); SYCEXP(end));
5160
4/6
✓ Branch 0 taken 57 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 27 times.
✓ Branch 3 taken 30 times.
✓ Branch 4 taken 27 times.
✗ Branch 5 not taken.
 57 if (synode_gt(end, max_synode)) set_max_synode(end);
5161
5162
1/2
✓ Branch 0 taken 57 times.
✗ Branch 1 not taken.
 57 free_forced_config_site_def();
5163 57 wait_forced_config = 0;
5164 57 forced_config = s;
5165
1/2
✓ Branch 0 taken 57 times.
✗ Branch 1 not taken.
 57 force_interval(executed_msg, max_synode,
5166 enforcer); /* Force everything in the pipeline */
5167 57 }
5168
5169 /* Learn this value */
5170 497973 void handle_learn(site_def const *site, pax_machine *p, pax_msg *m) {
5171 IFDBG(D_NONE, FN; STRLIT("proposer nodeset ");
5172 dbg_bitset(p->proposer.prop_nodeset, get_maxnodes(site)););
5173 IFDBG(D_NONE, FN; STRLIT("receivers ");
5174 dbg_bitset(m->receivers, get_maxnodes(site)););
5175 IFDBG(D_NONE, FN; NDBG(task_now(), f); SYCEXP(p->synode);
5176 COPY_AND_FREE_GOUT(dbg_app_data(m->a)););
5177
5178 PAX_MSG_SANITY_CHECK(m);
5179 497973 p->last_modified = task_now();
5180
2/2
✓ Branch 0 taken 344274 times.
✓ Branch 1 taken 153699 times.
 497973 if (!finished(p)) { /* Avoid re-learn */
5181 344274 activate_sweeper();
5182 344274 do_learn(site, p, m);
5183 /* Check for special messages */
5184
4/4
✓ Branch 0 taken 217419 times.
✓ Branch 1 taken 126855 times.
✓ Branch 2 taken 837 times.
✓ Branch 3 taken 216582 times.
 344274 if (m->a && m->a->body.c_t == unified_boot_type) {
5185 IFDBG(D_NONE, FN; STRLIT("Got unified_boot "); SYCEXP(p->synode);
5186 SYCEXP(m->synode););
5187
4/4
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 836 times.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 836 times.
 837 XCOM_FSM(x_fsm_net_boot, void_arg(m->a));
5188 }
5189 /* See if someone is forcing a new config */
5190
3/4
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 344194 times.
✓ Branch 2 taken 80 times.
✗ Branch 3 not taken.
 344274 if (m->force_delivery && m->a) {
5191 IFDBG(D_NONE, FN; STRLIT("Got forced config "); SYCEXP(p->synode);
5192 SYCEXP(m->synode););
5193 /* Configure all messages from executed_msg until start of new config
5194 as forced messages so they will eventually be finished */
5195 /* Immediately install this new config */
5196
2/4
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✓ Branch 2 taken 30 times.
✓ Branch 3 taken 50 times.
 80 switch (m->a->body.c_t) {
5197 ✗ case add_node_type:
5198 /* purecov: begin deadcode */
5199 ✗ if (should_ignore_forced_config_or_view(
5200 ✗ find_site_def(p->synode)->x_proto)) {
5201 ✗ log_ignored_forced_config(m->a, "handle_learn");
5202 } else {
5203 ✗ site_def *new_def = handle_add_node(m->a);
5204 ✗ if (new_def) start_force_config(clone_site_def(new_def), 0);
5205 }
5206 ✗ break;
5207 /* purecov: end */
5208 ✗ case remove_node_type:
5209 /* purecov: begin deadcode */
5210 ✗ if (should_ignore_forced_config_or_view(
5211 ✗ find_site_def(p->synode)->x_proto)) {
5212 ✗ log_ignored_forced_config(m->a, "handle_learn");
5213 } else {
5214 ✗ start_force_config(clone_site_def(handle_remove_node(m->a)), 0);
5215 }
5216 ✗ break;
5217 /* purecov: end */
5218 30 case force_config_type:
5219 30 start_force_config(clone_site_def(install_node_group(m->a)), 0);
5220 30 break;
5221 50 default:
5222 50 break;
5223 }
5224 }
5225 }
5226
5227 497973 task_wakeup(&p->rv);
5228 497973 }
5229
5230 /* Skip this value */
5231 256248 static void handle_skip(site_def const *site, pax_machine *p, pax_msg *m) {
5232 /* IFDBG(D_NONE, FN;); */
5233 /* IFDBG(D_NONE, FN; NDBG(task_now(),f); SYCEXP(p->msg->synode)); */
5234
2/2
✓ Branch 0 taken 255176 times.
✓ Branch 1 taken 1072 times.
 256248 if (!finished(p)) {
5235 255176 p->last_modified = task_now();
5236 255176 skip_value(m);
5237 255176 do_learn(site, p, m);
5238 }
5239 /* IFDBG(D_NONE, FN; STRLIT("taskwakeup "); SYCEXP(p->msg->synode)); */
5240 256248 task_wakeup(&p->rv);
5241 256248 }
5242
5243 107464 static void handle_client_msg(pax_msg *p) {
5244
2/4
✓ Branch 0 taken 107464 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 107464 times.
 107464 if (!p || p->a == nullptr) /* discard invalid message */
5245 ✗ return;
5246 {
5247 107464 msg_link *ml = msg_link_new(p, VOID_NODE_NO);
5248
5249 /* Put it in the proposer queue */
5250 ADD_T_EV(task_now(), __FILE__, __LINE__, "handle_client_msg");
5251 107464 channel_put(&prop_input_queue, &ml->l);
5252 }
5253 }
5254
5255 #ifdef ACCEPT_SITE_TEST
5256 /* See if we should process an incoming ping from a node.
5257 The purpose is to avoid doing recovery from a node with an obsolete site
5258 definition */
5259 static int accept_site(site_def const *site) {
5260 site_def *mysite = (site_def *)get_site_def();
5261
5262 if (site) {
5263 if (!mysite) {
5264 site_def *prev = (site_def *)find_prev_site_def(site->boot_key);
5265 IFDBG(
5266 D_NONE, FN; PTREXP(site); PTREXP(mysite); PTREXP(prev);
5267 SYCEXP(site->boot_key); if (prev) { SYCEXP(prev->boot_key); });
5268 if (!prev) {
5269 /** alive when no site, no known previous definition, and present in
5270 * new is accepted */
5271 return (site->boot_key.group_id == 0
5272 ? 1
5273 : (xcom_find_node_index((node_list *)&site->nodes) !=
5274 VOID_NODE_NO));
5275 } else {
5276 /** alive when no site, a previous definition of groupid is known, but
5277 * is older than site def, is accepted */
5278 return synode_gt(site->boot_key, prev->boot_key);
5279 }
5280 } else {
5281 IFDBG(D_NONE, FN; PTREXP(site); PTREXP(mysite); SYCEXP(site->boot_key);
5282 SYCEXP(mysite->boot_key));
5283 if (get_group_id(site) != get_group_id(mysite)) {
5284 /** alive from different site should never be accepted */
5285 return 0;
5286 } else {
5287 /** alive from same site should be accepted if boot_key is larger than
5288 * mine */
5289 node_no my_nodeno = xcom_find_node_index((node_list *)&mysite->nodes);
5290 node_no site_nodeno = xcom_find_node_index((node_list *)&site->nodes);
5291 return (synode_gt(site->boot_key, mysite->boot_key) &&
5292 ((my_nodeno != VOID_NODE_NO) || (site_nodeno != VOID_NODE_NO)));
5293 }
5294 }
5295 }
5296 /** Always accept a NULL site */
5297 IFDBG(D_NONE, FN; PTREXP(site));
5298 return 1;
5299 }
5300 #endif
5301
5302 /* Handle incoming "need boot" message. */
5303 /* purecov: begin deadcode */
5304 ✗ static inline void handle_boot(site_def const *site, linkage *reply_queue,
5305 pax_msg *p) {
5306 /* This should never be TRUE, but validate it instead of asserting. */
5307 ✗ if (site == nullptr || site->nodes.node_list_len < 1) {
5308 ✗ G_DEBUG(
5309 "handle_boot: Received an unexpected need_boot_op when site == NULL "
5310 "or "
5311 "site->nodes.node_list_len < 1");
5312 ✗ return;
5313 }
5314
5315 ✗ if (ALWAYS_HANDLE_NEED_BOOT || should_handle_need_boot(site, p)) {
5316 ✗ handle_need_snapshot(reply_queue, p);
5317 } else {
5318 ✗ G_DEBUG(
5319 "Ignoring a need_boot_op message from an XCom incarnation that does "
5320 "not belong to the group.");
5321 }
5322 }
5323 /* purecov: end */
5324
5325 1324 bool_t should_handle_need_boot(site_def const *site, pax_msg *p) {
5326 1324 bool_t should_handle = FALSE;
5327 1324 bool_t const sender_advertises_identity =
5328
3/4
✓ Branch 0 taken 1321 times.
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 1321 times.
✗ Branch 3 not taken.
 1324 (p->a != nullptr && p->a->body.c_t == xcom_boot_type);
5329
5330 /*
5331 If the message advertises the sender's identity, check if it matches the
5332 membership information.
5333
5334 The sender's identity may not match if, e.g.:
5335
5336 a. The member was already removed, or
5337 b. It is a new incarnation of a crashed member that is yet to be removed.
5338
5339 ...or some other reason.
5340
5341 If it is due to reason (b), we do not want to boot the sender because XCom
5342 only implements a simple fail-stop model. Allowing the sender to rejoin the
5343 group without going through the full remove+add node path could violate
5344 safety because the sender does not remember any previous Paxos acceptances
5345 it acknowledged before crashing. Since the pre-crash incarnation may have
5346 accepted a value for a given synod but the post-crash incarnation has
5347 forgotten that fact, the post-crash incarnation will fail to propagate the
5348 previously accepted value to a higher ballot. Since majorities can overlap
5349 on a single node, if the overlap node is the post-crash incarnation which
5350 has forgotten about the previously accepted value, a higher ballot proposer
5351 may get a different value accepted, leading to conflicting values to be
5352 accepted for different proposers, which is a violation of the safety
5353 properties of the Paxos protocol.
5354
5355 If the sender does not advertise its identity, we boot it unconditionally.
5356 This is for backwards compatibility.
5357 */
5358
2/2
✓ Branch 0 taken 1321 times.
✓ Branch 1 taken 3 times.
 1324 if (sender_advertises_identity) {
5359 1321 bool_t const sender_advertises_one_identity =
5360 1321 (p->a->body.app_u_u.nodes.node_list_len == 1);
5361
5362 /* Defensively accept only messages with a single identity. */
5363
2/2
✓ Branch 0 taken 1318 times.
✓ Branch 1 taken 3 times.
 1321 if (sender_advertises_one_identity) {
5364 1318 node_address *sender_identity = p->a->body.app_u_u.nodes.node_list_val;
5365
5366 1318 should_handle = node_exists_with_uid(sender_identity, &site->nodes);
5367 }
5368 } else {
5369 3 should_handle = TRUE;
5370 }
5371
5372 1324 return should_handle;
5373 }
5374
5375 1369 void init_need_boot_op(pax_msg *p, node_address *identity) {
5376 1369 p->op = need_boot_op;
5377
2/2
✓ Branch 0 taken 1366 times.
✓ Branch 1 taken 3 times.
 1369 if (identity != nullptr) {
5378 1366 p->a = new_app_data();
5379 1366 p->a->body.c_t = xcom_boot_type;
5380 1366 init_node_list(1, identity, &p->a->body.app_u_u.nodes);
5381 }
5382 1369 }
5383
5384 #define PING_GATHERING_TIME_WINDOW 5.0
5385 #define PINGS_GATHERED_BEFORE_CONNECTION_SHUTDOWN 3
5386
5387 399300 int pre_process_incoming_ping(site_def const *site, pax_msg const *pm,
5388 int has_client_already_booted,
5389 double current_time) {
5390 // Yes... it is a ping for me, boot is done and it is a are_you_alive_op
5391 // This means that something wrong is not right...
5392 399300 int did_shutdown = 0;
5393
5394
6/6
✓ Branch 0 taken 201317 times.
✓ Branch 1 taken 197983 times.
✓ Branch 2 taken 196442 times.
✓ Branch 3 taken 4875 times.
✓ Branch 4 taken 3536 times.
✓ Branch 5 taken 395764 times.
 595742 if ((pm->from != get_nodeno(site)) && has_client_already_booted &&
5395
2/2
✓ Branch 0 taken 3536 times.
✓ Branch 1 taken 192906 times.
 196442 (pm->op == are_you_alive_op)) {
5396
2/2
✓ Branch 0 taken 49 times.
✓ Branch 1 taken 3487 times.
 3536 G_DEBUG(
5397 "Received a ping to myself. This means that something must be wrong "
5398 "in "
5399 "a bi-directional connection")
5400 // Going to kill the connection for that node...
5401
2/4
✓ Branch 0 taken 3536 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3536 times.
✗ Branch 3 not taken.
 3536 if (site && (pm->from < site->nodes.node_list_len)) {
5402 // This is not the first ping received in the last 5 seconds...
5403 3536 if (site->servers[pm->from]->last_ping_received >
5404
2/2
✓ Branch 0 taken 1365 times.
✓ Branch 1 taken 2171 times.
 3536 (current_time - PING_GATHERING_TIME_WINDOW)) {
5405 1365 site->servers[pm->from]->number_of_pings_received++;
5406 } else { // First ping since at least more than 5 seconds...
5407 2171 site->servers[pm->from]->number_of_pings_received = 1;
5408 }
5409
5410 3536 site->servers[pm->from]->last_ping_received = current_time;
5411
5412 // If we keep on receiving periodical pings... lets kill the connection
5413
4/4
✓ Branch 0 taken 2473 times.
✓ Branch 1 taken 1063 times.
✓ Branch 2 taken 121 times.
✓ Branch 3 taken 3415 times.
 6009 if (is_connected(site->servers[pm->from]->con) &&
5414
2/2
✓ Branch 0 taken 121 times.
✓ Branch 1 taken 2352 times.
 2473 site->servers[pm->from]->number_of_pings_received ==
5415 PINGS_GATHERED_BEFORE_CONNECTION_SHUTDOWN) {
5416 121 shutdown_connection(site->servers[pm->from]->con);
5417
2/4
✓ Branch 0 taken 121 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 121 times.
✗ Branch 3 not taken.
 121 G_WARNING(
5418 "Shutting down an outgoing connection. This happens because "
5419 "something might be wrong on a bi-directional connection to node "
5420 "%s:%d. Please check the connection status to this member",
5421 site->servers[pm->from]->srv, site->servers[pm->from]->port);
5422 121 did_shutdown = 1;
5423 }
5424 }
5425 }
5426
5427 399300 return did_shutdown;
5428 }
5429
5430 /* Handle incoming alive message */
5431 static double sent_alive = 0.0;
5432 399240 static inline void handle_alive(site_def const *site, linkage *reply_queue,
5433 pax_msg *pm) {
5434 399240 pre_process_incoming_ping(site, pm, client_boot_done, task_now());
5435
5436
6/6
✓ Branch 0 taken 4863 times.
✓ Branch 1 taken 394377 times.
✓ Branch 2 taken 3503 times.
✓ Branch 3 taken 1360 times.
✓ Branch 4 taken 397880 times.
✓ Branch 5 taken 1360 times.
 399240 if (client_boot_done || !(task_now() - sent_alive > 1.0)) /* Already done? */
5437 397880 return;
5438
5439 #ifdef ACCEPT_SITE_TEST
5440 if (!accept_site(site)) return;
5441 #endif
5442
5443 /* Avoid responding to own ping */
5444
3/6
✓ Branch 0 taken 1360 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1360 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 1360 times.
 1360 if (pm->from == get_nodeno(site) || pm->from == pm->to) return;
5445
5446 /*
5447 This code will check if the ping is intended to us.
5448 If the encoded node does not exist in the current configuration,
5449 we avoid sending need_boot_op, since it must be from a different
5450 reincarnation of this node.
5451 */
5452
1/6
✗ Branch 0 not taken.
✓ Branch 1 taken 1360 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 1360 if (site && pm->a && pm->a->body.c_t == xcom_boot_type) {
5453 IFDBG(D_NONE, FN;
5454 COPY_AND_FREE_GOUT(dbg_list(&pm->a->body.app_u_u.nodes)););
5455
5456 ✗ if (!node_exists_with_uid(&pm->a->body.app_u_u.nodes.node_list_val[0],
5457 ✗ &get_site_def()->nodes))
5458 ✗ return;
5459 }
5460
5461
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1360 times.
 1360 if (is_dead_site(pm->group_id)) return; /* Avoid dealing with zombies */
5462
5463 {
5464
2/4
✓ Branch 0 taken 1360 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1360 times.
✗ Branch 3 not taken.
 1360 CREATE_REPLY(pm);
5465
2/4
✓ Branch 0 taken 1360 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1360 times.
✗ Branch 3 not taken.
 1360 init_need_boot_op(reply, cfg_app_xcom_get_identity());
5466
1/2
✓ Branch 0 taken 1360 times.
✗ Branch 1 not taken.
 1360 sent_alive = task_now();
5467
2/4
✓ Branch 0 taken 1360 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1360 times.
✗ Branch 3 not taken.
 1360 G_INFO(
5468 "Node has not booted. Requesting an XCom snapshot from node number %d "
5469 "in the current configuration",
5470 pm->from);
5471
4/10
✓ Branch 0 taken 1360 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1360 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 1360 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 1360 times.
✗ Branch 9 not taken.
 1360 SEND_REPLY;
5472 }
5473 IFDBG(D_NONE, FN; STRLIT("sent need_boot_op"););
5474 }
5475
5476 499327 static void update_max_synode(pax_msg *p) {
5477
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 499324 times.
 499327 if (is_dead_site(p->group_id)) return;
5478
5/6
✓ Branch 0 taken 477083 times.
✓ Branch 1 taken 22241 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 477083 times.
✓ Branch 4 taken 22241 times.
✓ Branch 5 taken 477083 times.
 499324 if (get_group_id(get_site_def()) == 0 || max_synode.group_id == 0) {
5479 22241 set_max_synode(p->synode);
5480
1/2
✓ Branch 0 taken 477083 times.
✗ Branch 1 not taken.
 477083 } else if (max_synode.group_id == p->synode.group_id) {
5481
2/2
✓ Branch 0 taken 203449 times.
✓ Branch 1 taken 273634 times.
 477083 if (synode_gt(p->synode, max_synode)) {
5482 203449 set_max_synode(p->synode);
5483 }
5484
2/2
✓ Branch 0 taken 3786 times.
✓ Branch 1 taken 473297 times.
 477083 if (synode_gt(p->max_synode, max_synode)) {
5485 3786 set_max_synode(p->max_synode);
5486 }
5487 }
5488 }
5489
5490 /* Message dispatch */
5491 #define BAL_FMT "ballot {cnt %d node %d}"
5492 #define BAL_MEM(x) (x).cnt, (x).node
5493
5494 static int clicnt = 0;
5495
5496 23 xcom_event_horizon xcom_get_minimum_event_horizon() {
5497 23 return EVENT_HORIZON_MIN;
5498 }
5499
5500 23 xcom_event_horizon xcom_get_maximum_event_horizon() {
5501 23 return EVENT_HORIZON_MAX;
5502 }
5503
5504 /**
5505 * Retrieves the latest event horizon.
5506 *
5507 * There is no specific reason for this method to return the latest event
5508 * horizon instead of the current one. Both would be acceptable results of
5509 * this function, but we had to make a decision of one over the other.
5510 *
5511 * @param[out] event_horizon the latest event horizon
5512 * @retval REQUEST_FAIL XCom is not initialized yet
5513 * @retval REQUEST_OK function was successful and event_horizon contains the
5514 * latest event horizon
5515 */
5516 99 static client_reply_code xcom_get_event_horizon(
5517 xcom_event_horizon *event_horizon) {
5518 99 site_def const *latest_config = get_site_def();
5519
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 99 times.
 99 if (latest_config == nullptr) return REQUEST_FAIL;
5520 99 *event_horizon = latest_config->event_horizon;
5521 99 return REQUEST_OK;
5522 }
5523
5524 1673 static u_int allow_add_node(app_data_ptr a) {
5525 /* Get information on the current site definition */
5526 1673 const site_def *new_site_def = get_site_def();
5527 1673 const site_def *valid_site_def = find_site_def(executed_msg);
5528
5529 /* Get information on the nodes to be added */
5530 1673 u_int nr_nodes_to_add = a->body.app_u_u.nodes.node_list_len;
5531 1673 node_address *nodes_to_change = a->body.app_u_u.nodes.node_list_val;
5532
5533
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1673 times.
 1673 if (check_if_add_node_is_unsafe_against_event_horizon(a)) return 0;
5534
5535
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1673 times.
 1673 if (unsafe_leaders(a)) {
5536 ✗ return 0;
5537 }
5538
5539
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1673 times.
 1673 if (add_node_unsafe_against_ipv4_old_nodes(a)) {
5540 ✗ G_MESSAGE(
5541 "This server is unable to join the group as the NIC used is "
5542 "configured "
5543 "with IPv6 only and there are members in the group that are unable "
5544 "to "
5545 "communicate using IPv6, only IPv4.Please configure this server to "
5546 "join the group using an IPv4 address instead.");
5547 ✗ return 0;
5548 }
5549
5550 {
5551 u_int i;
5552
2/2
✓ Branch 0 taken 1673 times.
✓ Branch 1 taken 1313 times.
 2986 for (i = 0; i < nr_nodes_to_add; i++) {
5553
4/4
✓ Branch 0 taken 1313 times.
✓ Branch 1 taken 360 times.
✓ Branch 2 taken 360 times.
✓ Branch 3 taken 1313 times.
 2986 if (node_exists(&nodes_to_change[i], &new_site_def->nodes) ||
5554
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1313 times.
 1313 node_exists(&nodes_to_change[i], &valid_site_def->nodes)) {
5555 /*
5556 We are simply ignoring the attempt to add a node to the
5557 group when there is an old incarnation of it, meaning
5558 that the node has crashed and restarted so fastly that
5559 nobody has noticed that it has gone.
5560
5561 In XCOM, the group is not automatically reconfigured
5562 and it is possible to start reusing a node that has
5563 crashed and restarted without reconfiguring the group
5564 by adding the node back to it.
5565
5566 However, this operation may be unsafe because XCOM
5567 does not implement a crash-recovery model and nodes
5568 suffer from amnesia after restarting the service. In
5569 other words this may lead to inconsistency issues in
5570 the paxos protocol.
5571
5572 Unfortunately, preventing that a node is added back
5573 to the system where there is an old incarnation will
5574 not fix this problem since other changes are required.
5575 */
5576
2/4
✓ Branch 0 taken 360 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 360 times.
✗ Branch 3 not taken.
 360 G_WARNING(
5577 "Old incarnation found while trying to "
5578 "add node %s %.*s. Please stop the old node or wait for it to "
5579 "leave the group.",
5580 nodes_to_change[i].address, nodes_to_change[i].uuid.data.data_len,
5581 nodes_to_change[i].uuid.data.data_val);
5582 360 return 0;
5583 }
5584 }
5585 }
5586
5587 1313 return 1;
5588 }
5589
5590 2293 static u_int allow_remove_node(app_data_ptr a) {
5591 /* Get information on the current site definition */
5592 2293 const site_def *new_site_def = get_site_def();
5593
5594 /* Get information on the nodes to be added */
5595 2293 u_int nodes_len = a->body.app_u_u.nodes.node_list_len;
5596 2293 node_address *nodes_to_change = a->body.app_u_u.nodes.node_list_val;
5597
5598 u_int i;
5599
2/2
✓ Branch 0 taken 2293 times.
✓ Branch 1 taken 2245 times.
 4538 for (i = 0; i < nodes_len; i++) {
5600
2/2
✓ Branch 0 taken 48 times.
✓ Branch 1 taken 2245 times.
 2293 if (!node_exists_with_uid(&nodes_to_change[i], &new_site_def->nodes)) {
5601 /*
5602 If the UID does not exist, then 1) the node has already been
5603 removed or 2) it has reincarnated.
5604 */
5605 /* purecov: begin inspected */
5606
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 48 times.
 48 if (node_exists(&nodes_to_change[i], &new_site_def->nodes)) {
5607 /*
5608 We also cannot allow an upper-layer to remove a new incarnation
5609 of a node when it tries to remove an old one.
5610 */
5611 ✗ G_MESSAGE(
5612 "New incarnation found while trying to "
5613 "remove node %s %.*s.",
5614 nodes_to_change[i].address, nodes_to_change[i].uuid.data.data_len,
5615 nodes_to_change[i].uuid.data.data_val);
5616 } else {
5617 /* The node has already been removed, so we block the request */
5618
2/4
✓ Branch 0 taken 48 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 48 times.
✗ Branch 3 not taken.
 48 G_MESSAGE(
5619 "Node has already been removed: "
5620 "%s %.*s.",
5621 nodes_to_change[i].address, nodes_to_change[i].uuid.data.data_len,
5622 nodes_to_change[i].uuid.data.data_val);
5623 }
5624 48 return 0;
5625 /* purecov: end */
5626 }
5627 }
5628
5629 2245 return 1;
5630 }
5631
5632 /**
5633 * Logs the fact that an add/remove node request is aimed at another group.
5634 *
5635 * @param a a pointer to the app_data of the configuration command
5636 * @param message_fmt a formatted message to log, containing a single %s that
5637 * will be replaced by the node's address
5638 */
5639 92 static void log_cfgchange_wrong_group(app_data_ptr a,
5640 const char *const message_fmt) {
5641 92 u_int const nr_nodes = a->body.app_u_u.nodes.node_list_len;
5642 u_int i;
5643
2/2
✓ Branch 0 taken 92 times.
✓ Branch 1 taken 92 times.
 184 for (i = 0; i < nr_nodes; i++) {
5644 92 char const *const address = a->body.app_u_u.nodes.node_list_val[i].address;
5645
2/4
✓ Branch 0 taken 92 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 92 times.
✗ Branch 3 not taken.
 92 G_WARNING(message_fmt, address);
5646 }
5647 92 }
5648
5649 /**
5650 * Validates if a configuration command can be executed.
5651 * Checks whether the configuration command is aimed at the correct group.
5652 * Checks whether the configuration command pertains to a node reincarnation.
5653 *
5654 * @param p a pointer to the pax_msg of the configuration command
5655 * @retval REQUEST_OK if the reconfiguration command can be executed
5656 * @retval REQUEST_RETRY if XCom is still booting
5657 * @retval REQUEST_FAIL if the configuration command cannot be executed
5658 */
5659 4134 static client_reply_code can_execute_cfgchange(pax_msg *p) {
5660 4134 app_data_ptr a = p->a;
5661
5662
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4134 times.
 4134 if (executed_msg.msgno <= 2) {
5663 // If we have not booted and we receive an add_node that contains us...
5664 ✗ if (add_node_adding_own_address(a))
5665 ✗ return REQUEST_FAIL;
5666 else {
5667 /*G_INFO(
5668 "Configuration change failed. Request on a node that has not booted "
5669 "yet.");*/
5670 ✗ G_INFO(
5671 "This node received a Configuration change request, but it not yet "
5672 "started. This could happen if one starts several nodes "
5673 "simultaneously. This request will be retried by whoever sent it.");
5674 ✗ return REQUEST_RETRY;
5675 }
5676 }
5677
5678
4/6
✓ Branch 0 taken 4134 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4134 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 92 times.
✓ Branch 5 taken 4042 times.
 4134 if (a && a->group_id != 0 && a->group_id != executed_msg.group_id) {
5679
1/6
✓ Branch 0 taken 92 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 92 switch (a->body.c_t) {
5680 92 case add_node_type:
5681 92 log_cfgchange_wrong_group(
5682 a,
5683 "The request to add %s to the group has been rejected because it "
5684 "is aimed at another group");
5685 92 break;
5686 ✗ case remove_node_type:
5687 ✗ log_cfgchange_wrong_group(a,
5688 "The request to remove %s from the group "
5689 "has been rejected because "
5690 "it is aimed at another group");
5691 ✗ break;
5692 ✗ case force_config_type:
5693 ✗ G_WARNING(
5694 "The request to force the group membership has been rejected "
5695 "because it is aimed at another group");
5696 ✗ break;
5697 ✗ case set_max_leaders:
5698 ✗ G_WARNING(
5699 "The request to change max number of leaders has been rejected "
5700 "because it is aimed at another group");
5701 ✗ break;
5702 ✗ case set_leaders_type:
5703 ✗ G_WARNING(
5704 "The request to change leaders has been rejected "
5705 "because it is aimed at another group");
5706 ✗ break;
5707 ✗ default:
5708 ✗ assert(0 &&
5709 "A cargo_type different from {add_node_type, remove_node_type, "
5710 "force_config_type, set_max_leaders, set_leaders_type} should "
5711 "not "
5712 "have hit this code path");
5713 }
5714 92 return REQUEST_FAIL;
5715 }
5716
5717
7/8
✓ Branch 0 taken 4042 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1673 times.
✓ Branch 3 taken 2369 times.
✓ Branch 4 taken 360 times.
✓ Branch 5 taken 1313 times.
✓ Branch 6 taken 360 times.
✓ Branch 7 taken 3682 times.
 4042 if (a && a->body.c_t == add_node_type && !allow_add_node(a))
5718 360 return REQUEST_FAIL;
5719
5720
7/8
✓ Branch 0 taken 3682 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2293 times.
✓ Branch 3 taken 1389 times.
✓ Branch 4 taken 48 times.
✓ Branch 5 taken 2245 times.
✓ Branch 6 taken 48 times.
✓ Branch 7 taken 3634 times.
 3682 if (a && a->body.c_t == remove_node_type && !allow_remove_node(a))
5721 48 return REQUEST_FAIL;
5722
5723
4/6
✓ Branch 0 taken 3634 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 9 times.
✓ Branch 3 taken 3625 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 3634 times.
 3643 if (a && a->body.c_t == set_event_horizon_type &&
5724
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 is_unsafe_event_horizon_reconfiguration(a))
5725 ✗ return REQUEST_FAIL;
5726
5727
5/6
✓ Branch 0 taken 3634 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 28 times.
✓ Branch 3 taken 3606 times.
✓ Branch 4 taken 1 times.
✓ Branch 5 taken 3633 times.
 3662 if (a && a->body.c_t == force_config_type &&
5728
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 27 times.
 28 are_there_dead_nodes_in_new_config(a))
5729 1 return REQUEST_FAIL;
5730
5731 7266 if (a &&
5732
5/8
✓ Branch 0 taken 3633 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3633 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 39 times.
✓ Branch 5 taken 3594 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 3633 times.
 3672 (a->body.c_t == set_max_leaders || a->body.c_t == set_leaders_type) &&
5733
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 39 times.
 39 is_unsafe_leaders_reconfiguration(a))
5734 ✗ return REQUEST_FAIL;
5735
5736 3633 return REQUEST_OK;
5737 }
5738
5739 831042 static void activate_sweeper() {
5740
2/2
✓ Branch 0 taken 783355 times.
✓ Branch 1 taken 47687 times.
 831042 if (sweeper) {
5741 ADD_DBG(D_CONS, add_event(EVENT_DUMP_PAD,
5742 string_arg("sweeper activated max_synode"));
5743 add_synode_event(max_synode););
5744 783355 task_activate(sweeper);
5745 }
5746 831042 }
5747
5748 static synode_no start_config = NULL_SYNODE;
5749
5750 99 void dispatch_get_event_horizon(site_def const *site, pax_msg *p,
5751 linkage *reply_queue) {
5752
2/4
✓ Branch 0 taken 99 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 99 times.
✗ Branch 3 not taken.
 99 CREATE_REPLY(p);
5753 IFDBG(D_NONE, FN; STRLIT("Got get_event_horizon from client");
5754 SYCEXP(p->synode););
5755 99 reply->op = xcom_client_reply;
5756
1/2
✓ Branch 0 taken 99 times.
✗ Branch 1 not taken.
 99 reply->cli_err = xcom_get_event_horizon(&reply->event_horizon);
5757
4/10
✓ Branch 0 taken 99 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 99 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 99 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 99 times.
✗ Branch 9 not taken.
 99 SEND_REPLY;
5758 99 }
5759
5760 85 static reply_data *new_leader_info(site_def *site) {
5761
1/2
✓ Branch 0 taken 85 times.
✗ Branch 1 not taken.
 85 if (site) {
5762 reply_data *data =
5763 85 static_cast<reply_data *>(xcom_calloc((size_t)1, sizeof(reply_data)));
5764 85 data->rt = leader_info;
5765 85 data->reply_data_u.leaders.max_nr_leaders = site->max_active_leaders;
5766
2/2
✓ Branch 0 taken 32 times.
✓ Branch 1 taken 53 times.
 85 if (leaders_set_by_client(site)) {
5767 data->reply_data_u.leaders.preferred_leaders =
5768 32 clone_leader_array(site->leaders);
5769 }
5770 85 active_leaders(site, &data->reply_data_u.leaders.actual_leaders);
5771 85 return data;
5772 } else {
5773 ✗ return nullptr;
5774 }
5775 }
5776
5777 85 void dispatch_get_leaders(site_def *site, pax_msg *p, linkage *reply_queue) {
5778
2/4
✓ Branch 0 taken 85 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 85 times.
✗ Branch 3 not taken.
 85 CREATE_REPLY(p);
5779 IFDBG(D_NONE, FN; STRLIT("Got get_leaders from client"); SYCEXP(p->synode););
5780 85 reply->op = xcom_client_reply;
5781
1/2
✓ Branch 0 taken 85 times.
✗ Branch 1 not taken.
 85 reply->rd = new_leader_info(site);
5782 85 reply->cli_err = reply->rd ? REQUEST_OK : REQUEST_FAIL;
5783
4/10
✓ Branch 0 taken 85 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 85 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 85 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 85 times.
✗ Branch 9 not taken.
 85 SEND_REPLY;
5784 85 }
5785
5786 /*
5787 * Log the result of the get_synode_app_data command.
5788 */
5789 ✗ static void log_get_synode_app_data_failure(
5790 xcom_get_synode_app_data_result error_code) {
5791 ✗ switch (error_code) {
5792 ✗ case XCOM_GET_SYNODE_APP_DATA_OK:
5793 ✗ break;
5794 ✗ case XCOM_GET_SYNODE_APP_DATA_ERROR:
5795 ✗ G_DEBUG("Could not reply successfully to request for synode data.");
5796 ✗ break;
5797 ✗ case XCOM_GET_SYNODE_APP_DATA_NOT_CACHED:
5798 ✗ G_DEBUG(
5799 "Could not reply successfully to request for synode data because "
5800 "some of the requested synodes are no longer cached.");
5801 ✗ break;
5802 ✗ case XCOM_GET_SYNODE_APP_DATA_NOT_DECIDED:
5803 ✗ G_DEBUG(
5804 "Could not reply successfully to request for synode data because "
5805 "some of the requested synodes are still undecided.");
5806 ✗ break;
5807 ✗ case XCOM_GET_SYNODE_APP_DATA_NO_MEMORY:
5808 ✗ G_DEBUG(
5809 "Could not reply successfully to request for synode data because "
5810 "memory could not be allocated.");
5811 ✗ break;
5812 }
5813 }
5814
5815 ✗ void dispatch_get_synode_app_data(site_def const *site, pax_msg *p,
5816 linkage *reply_queue) {
5817 IFDBG(D_NONE, FN; STRLIT("Got get_synode_app_data from client");
5818 SYCEXP(p->synode););
5819
5820 {
5821 ✗ CREATE_REPLY(p);
5822 ✗ reply->op = xcom_client_reply;
5823
5824 {
5825 xcom_get_synode_app_data_result error_code;
5826 ✗ error_code = xcom_get_synode_app_data(&p->a->body.app_u_u.synodes,
5827 ✗ &reply->requested_synode_app_data);
5828 ✗ switch (error_code) {
5829 ✗ case XCOM_GET_SYNODE_APP_DATA_OK:
5830 ✗ reply->cli_err = REQUEST_OK;
5831 ✗ break;
5832 ✗ case XCOM_GET_SYNODE_APP_DATA_NOT_CACHED:
5833 case XCOM_GET_SYNODE_APP_DATA_NOT_DECIDED:
5834 case XCOM_GET_SYNODE_APP_DATA_NO_MEMORY:
5835 case XCOM_GET_SYNODE_APP_DATA_ERROR:
5836 ✗ reply->cli_err = REQUEST_FAIL;
5837 ✗ log_get_synode_app_data_failure(error_code);
5838 ✗ break;
5839 }
5840
5841 ✗ SEND_REPLY;
5842 }
5843 }
5844 }
5845
5846 static int can_send_snapshot();
5847
5848 108152 static void process_client_msg(site_def const *site, pax_msg *p,
5849 linkage *reply_queue) {
5850 108152 clicnt++;
5851
2/4
✓ Branch 0 taken 108152 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 108152 times.
 108152 if (p->a && (p->a->body.c_t == exit_type)) {
5852 /* purecov: begin deadcode */
5853 IFDBG(D_NONE, FN; STRLIT("Got exit from client"); SYCEXP(p->synode););
5854 ✗ bury_site(get_group_id(get_site_def()));
5855 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("terminating"));)
5856 ✗ terminate_and_exit();
5857 ✗ return;
5858 /* purecov: end */
5859 }
5860
5861
2/4
✓ Branch 0 taken 108152 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 108152 times.
 108152 if (p->a && (p->a->body.c_t == reset_type)) {
5862 /* purecov: begin deadcode */
5863 IFDBG(D_NONE, FN; STRLIT("Got reset from client"); SYCEXP(p->synode););
5864 ✗ bury_site(get_group_id(get_site_def()));
5865 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("terminating"));)
5866 ✗ XCOM_FSM(x_fsm_terminate, int_arg(0));
5867 ✗ return;
5868 /* purecov: end */
5869 }
5870
2/4
✓ Branch 0 taken 108152 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 108152 times.
 108152 if (p->a && (p->a->body.c_t == remove_reset_type)) {
5871 /* purecov: begin deadcode */
5872 IFDBG(D_NONE, FN; STRLIT("Got remove_reset from client");
5873 SYCEXP(p->synode););
5874 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("terminating"));)
5875 ✗ XCOM_FSM(x_fsm_terminate, int_arg(0));
5876 ✗ return;
5877 /* purecov: end */
5878 }
5879
2/4
✓ Branch 0 taken 108152 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 108152 times.
 108152 if (p->a && (p->a->body.c_t == enable_arbitrator)) {
5880 ✗ CREATE_REPLY(p);
5881 IFDBG(D_NONE, FN; STRLIT("Got enable_arbitrator from client");
5882 SYCEXP(p->synode););
5883 ✗ ARBITRATOR_HACK = 1;
5884 ✗ reply->op = xcom_client_reply;
5885 ✗ reply->cli_err = REQUEST_OK;
5886 ✗ SEND_REPLY;
5887 ✗ return;
5888 }
5889
2/4
✓ Branch 0 taken 108152 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 108152 times.
 108152 if (p->a && (p->a->body.c_t == disable_arbitrator)) {
5890 ✗ CREATE_REPLY(p);
5891 IFDBG(D_NONE, FN; STRLIT("Got disable_arbitrator from client");
5892 SYCEXP(p->synode););
5893 ✗ ARBITRATOR_HACK = 0;
5894 ✗ reply->op = xcom_client_reply;
5895 ✗ reply->cli_err = REQUEST_OK;
5896 ✗ SEND_REPLY;
5897 ✗ return;
5898 }
5899
3/4
✓ Branch 0 taken 108152 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
✓ Branch 3 taken 108149 times.
 108152 if (p->a && (p->a->body.c_t == set_cache_limit)) {
5900
2/4
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
 3 CREATE_REPLY(p);
5901 IFDBG(D_NONE, FN; STRLIT("Got set_cache_limit from client");
5902 SYCEXP(p->synode););
5903
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 if (the_app_xcom_cfg) {
5904
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 set_max_cache_size(p->a->body.app_u_u.cache_limit);
5905 3 reply->cli_err = REQUEST_OK;
5906 } else {
5907 ✗ reply->cli_err = REQUEST_FAIL;
5908 }
5909 3 reply->op = xcom_client_reply;
5910
4/10
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 3 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 3 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 3 times.
✗ Branch 9 not taken.
 3 SEND_REPLY;
5911 3 return;
5912 }
5913
2/4
✓ Branch 0 taken 108149 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 108149 times.
 108149 if (p->a && (p->a->body.c_t == x_terminate_and_exit)) {
5914 /* purecov: begin deadcode */
5915 ✗ CREATE_REPLY(p);
5916 IFDBG(D_NONE, FN; STRLIT("Got terminate_and_exit from client");
5917 SYCEXP(p->synode););
5918 ✗ reply->op = xcom_client_reply;
5919 ✗ reply->cli_err = REQUEST_OK;
5920 ✗ SEND_REPLY;
5921 /*
5922 The function frees sites which is used by SEND_REPLY,
5923 so it should be called after SEND_REPLY.
5924 */
5925 IFDBG(D_NONE, FN; STRLIT("terminate_and_exit"));
5926 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("terminating"));)
5927 ✗ terminate_and_exit();
5928 ✗ return;
5929 /* purecov: end */
5930 }
5931
3/4
✓ Branch 0 taken 108149 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 99 times.
✓ Branch 3 taken 108050 times.
 108149 if (p->a && (p->a->body.c_t == get_event_horizon_type)) {
5932 99 dispatch_get_event_horizon(get_site_def(), p, reply_queue);
5933 99 return;
5934 }
5935
2/4
✓ Branch 0 taken 108050 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 108050 times.
 108050 if (p->a && (p->a->body.c_t == get_synode_app_data_type)) {
5936 ✗ dispatch_get_synode_app_data(get_site_def(), p, reply_queue);
5937 ✗ return;
5938 }
5939
3/4
✓ Branch 0 taken 108050 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 85 times.
✓ Branch 3 taken 107965 times.
 108050 if (p->a && (p->a->body.c_t == get_leaders_type)) {
5940 85 dispatch_get_leaders(get_site_def_rw(), p, reply_queue);
5941 85 return;
5942 }
5943
1/2
✓ Branch 0 taken 107965 times.
✗ Branch 1 not taken.
 107965 if (p->a &&
5944
4/4
✓ Branch 0 taken 106200 times.
✓ Branch 1 taken 1765 times.
✓ Branch 2 taken 103907 times.
✓ Branch 3 taken 2293 times.
 107965 (p->a->body.c_t == add_node_type || p->a->body.c_t == remove_node_type ||
5945
2/2
✓ Branch 0 taken 103879 times.
✓ Branch 1 taken 28 times.
 103907 p->a->body.c_t == force_config_type ||
5946
2/2
✓ Branch 0 taken 103870 times.
✓ Branch 1 taken 9 times.
 103879 p->a->body.c_t == set_event_horizon_type ||
5947
1/2
✓ Branch 0 taken 103870 times.
✗ Branch 1 not taken.
 103870 p->a->body.c_t == set_max_leaders ||
5948
2/2
✓ Branch 0 taken 39 times.
✓ Branch 1 taken 103831 times.
 103870 p->a->body.c_t == set_leaders_type)) {
5949 client_reply_code cli_err;
5950
2/4
✓ Branch 0 taken 4134 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4134 times.
✗ Branch 3 not taken.
 4134 CREATE_REPLY(p);
5951 4134 reply->op = xcom_client_reply;
5952
1/2
✓ Branch 0 taken 4134 times.
✗ Branch 1 not taken.
 4134 reply->cli_err = cli_err = can_execute_cfgchange(p);
5953
4/10
✓ Branch 0 taken 4134 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 4134 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 4134 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 4134 times.
✗ Branch 9 not taken.
 4134 SEND_REPLY;
5954
2/2
✓ Branch 0 taken 501 times.
✓ Branch 1 taken 3633 times.
 4134 if (cli_err != REQUEST_OK) {
5955 501 return;
5956 }
5957 }
5958
3/4
✓ Branch 0 taken 107464 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 837 times.
✓ Branch 3 taken 106627 times.
 107464 if (p->a && p->a->body.c_t == unified_boot_type) {
5959 IFDBG(D_NONE, FN; STRLIT("Got unified_boot from client");
5960 SYCEXP(p->synode););
5961 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&p->a->body.app_u_u.nodes)););
5962 IFDBG(D_NONE, STRLIT("handle_client_msg "); NDBG(p->a->group_id, x));
5963
4/4
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 836 times.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 836 times.
 837 XCOM_FSM(x_fsm_net_boot, void_arg(p->a));
5964 }
5965
3/4
✓ Branch 0 taken 107464 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1313 times.
✓ Branch 3 taken 106151 times.
 107464 if (p->a && p->a->body.c_t == add_node_type) {
5966 IFDBG(D_NONE, FN; STRLIT("Got add_node from client"); SYCEXP(p->synode););
5967 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&p->a->body.app_u_u.nodes)););
5968 IFDBG(D_NONE, STRLIT("handle_client_msg "); NDBG(p->a->group_id, x));
5969
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1313 times.
 1313 assert(get_site_def());
5970 }
5971
3/4
✓ Branch 0 taken 107464 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2245 times.
✓ Branch 3 taken 105219 times.
 107464 if (p->a && p->a->body.c_t == remove_node_type) {
5972 IFDBG(D_NONE, FN; STRLIT("Got remove_node from client");
5973 SYCEXP(p->synode););
5974 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&p->a->body.app_u_u.nodes)););
5975 IFDBG(D_NONE, STRLIT("handle_client_msg "); NDBG(p->a->group_id, x));
5976
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2245 times.
 2245 assert(get_site_def());
5977 }
5978
3/4
✓ Branch 0 taken 107464 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 9 times.
✓ Branch 3 taken 107455 times.
 107464 if (p->a && p->a->body.c_t == set_event_horizon_type) {
5979 IFDBG(D_NONE, FN; STRLIT("Got set_event_horizon from client");
5980 SYCEXP(p->synode););
5981 IFDBG(D_NONE, FN; NDBG(p->a->body.app_u_u.event_horizon, u));
5982 IFDBG(D_NONE, STRLIT("handle_client_msg "); NDBG(p->a->group_id, x));
5983
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 assert(get_site_def());
5984 }
5985
3/4
✓ Branch 0 taken 107464 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 27 times.
✓ Branch 3 taken 107437 times.
 107464 if (p->a && p->a->body.c_t == force_config_type) {
5986 IFDBG(D_NONE, FN; STRLIT("Got new force config from client");
5987 SYCEXP(p->synode););
5988 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&p->a->body.app_u_u.nodes)););
5989 IFDBG(D_NONE, STRLIT("handle_client_msg "); NDBG(p->a->group_id, x));
5990
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 27 times.
 27 assert(get_site_def());
5991
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 27 times.
 27 XCOM_FSM(x_fsm_force_config, void_arg(p->a));
5992 }
5993
2/4
✓ Branch 0 taken 107464 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 107464 times.
 107464 if (p->a && p->a->body.c_t == set_max_leaders) {
5994 IFDBG(D_NONE, FN; STRLIT("Got set_max_leaders from client");
5995 SYCEXP(p->synode););
5996 IFDBG(D_NONE, FN; NDBG(p->a->body.app_u_u.max_leaders, u));
5997 IFDBG(D_NONE, STRLIT("handle_client_msg "); NDBG(p->a->group_id, x));
5998 ✗ assert(get_site_def());
5999 }
6000
3/4
✓ Branch 0 taken 107464 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 39 times.
✓ Branch 3 taken 107425 times.
 107464 if (p->a && p->a->body.c_t == set_leaders_type) {
6001 IFDBG(D_NONE, FN; STRLIT("Got set_leaders_type from client");
6002 SYCEXP(p->synode););
6003 IFDBG(D_NONE, STRLIT("handle_client_msg "); NDBG(p->a->group_id, x));
6004
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 39 times.
 39 assert(get_site_def());
6005 }
6006 107464 handle_client_msg(p);
6007 }
6008
6009 53069 static void process_prepare_op(site_def const *site, pax_msg *p,
6010 linkage *reply_queue) {
6011 53069 pax_machine *pm = get_cache(p->synode);
6012
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 53069 times.
 53069 assert(pm);
6013
2/2
✓ Branch 0 taken 1760 times.
✓ Branch 1 taken 51309 times.
 53069 if (p->force_delivery) pm->force_delivery = 1;
6014 IFDBG(D_NONE, FN; dbg_pax_msg(p));
6015
6016 /*
6017 We can only be a productive Paxos Acceptor if we have been booted, i.e.
6018 added to the group and received an up-to-date snapshot from some member.
6019
6020 We do not allow non-booted members to participate in Paxos because they
6021 might be a reincarnation of a member that crashed and was then brought up
6022 without having gone through the remove+add node path.
6023 Since the pre-crash incarnation may have accepted a value for a given
6024 synod but the post-crash incarnation has forgotten that fact, the
6025 post-crash incarnation will fail to propagate the previously accepted
6026 value to a higher ballot. Since majorities can overlap on a single node,
6027 if the overlap node is the post-crash incarnation which has forgotten
6028 about the previously accepted value, the higher ballot proposer may get
6029 a different value accepted, leading to conflicting values to be accepted
6030 for different proposers, which is a violation of the safety requirements
6031 of the Paxos protocol.
6032 */
6033
2/2
✓ Branch 0 taken 39388 times.
✓ Branch 1 taken 13681 times.
 53069 if (ALWAYS_HANDLE_CONSENSUS || client_boot_done) {
6034 39388 paxos_fsm(pm, site, paxos_prepare, p);
6035 39388 handle_prepare(site, pm, reply_queue, p);
6036 }
6037 53069 }
6038
6039 36251 static inline int abort_processing(pax_msg *p) {
6040 /* Ensure that forced message can be processed */
6041
4/6
✓ Branch 0 taken 34491 times.
✓ Branch 1 taken 1760 times.
✓ Branch 2 taken 34491 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 36251 times.
 36251 return (!p->force_delivery && too_far(p->synode)) || !is_cached(p->synode);
6042 }
6043
6044 36251 static void process_ack_prepare_op(site_def const *site, pax_msg *p,
6045 linkage *reply_queue) {
6046 (void)reply_queue;
6047
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 36251 times.
 36251 if (abort_processing(p))
6048 ✗ return;
6049 else {
6050 36251 pax_machine *pm = get_cache(p->synode);
6051
2/2
✓ Branch 0 taken 1760 times.
✓ Branch 1 taken 34491 times.
 36251 if (p->force_delivery) pm->force_delivery = 1;
6052
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 36251 times.
 36251 if (!pm->proposer.msg) return;
6053
2/4
✓ Branch 0 taken 36251 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 36251 times.
✗ Branch 3 not taken.
 36251 assert(pm && pm->proposer.msg);
6054 36251 handle_ack_prepare(site, pm, p);
6055 36251 paxos_fsm(pm, site, paxos_ack_prepare, p);
6056 }
6057 }
6058
6059 242683 static void process_accept_op(site_def const *site, pax_msg *p,
6060 linkage *reply_queue) {
6061 242683 pax_machine *pm = get_cache(p->synode);
6062
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 242683 times.
 242683 assert(pm);
6063
2/2
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 242603 times.
 242683 if (p->force_delivery) pm->force_delivery = 1;
6064 IFDBG(D_NONE, FN; dbg_pax_msg(p));
6065
6066 /*
6067 We can only be a productive Paxos Acceptor if we have been booted, i.e.
6068 added to the group and received an up-to-date snapshot from some member.
6069
6070 We do not allow non-booted members to participate in Paxos because they
6071 might be a reincarnation of a member that crashed and was then brought up
6072 without having gone through the remove+add node path.
6073 Since the pre-crash incarnation may have accepted a value for a given
6074 synod but the post-crash incarnation has forgotten that fact, the
6075 post-crash incarnation will fail to propagate the previously accepted
6076 value to a higher ballot. Since majorities can overlap on a single node,
6077 if the overlap node is the post-crash incarnation which has forgotten
6078 about the previously accepted value, the higher ballot proposer may get
6079 a different value accepted, leading to conflicting values to be accepted
6080 for different proposers, which is a violation of the safety requirements
6081 of the Paxos protocol.
6082 */
6083
2/2
✓ Branch 0 taken 238990 times.
✓ Branch 1 taken 3693 times.
 242683 if (ALWAYS_HANDLE_CONSENSUS || client_boot_done) {
6084 238990 handle_alive(site, reply_queue, p);
6085
6086 238990 paxos_fsm(pm, site, paxos_accept, p);
6087 238990 handle_accept(site, pm, reply_queue, p);
6088 }
6089 242683 }
6090
6091 234831 static void process_ack_accept_op(site_def const *site, pax_msg *p,
6092 linkage *reply_queue) {
6093 (void)reply_queue;
6094
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 234831 times.
 234831 if (too_far(p->synode))
6095 ✗ return;
6096 else {
6097 234831 pax_machine *pm = get_cache(p->synode);
6098
2/2
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 234751 times.
 234831 if (p->force_delivery) pm->force_delivery = 1;
6099
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 234831 times.
 234831 if (!pm->proposer.msg) return;
6100
2/4
✓ Branch 0 taken 234831 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 234831 times.
✗ Branch 3 not taken.
 234831 assert(pm && pm->proposer.msg);
6101 234831 handle_ack_accept(site, pm, p);
6102 234831 paxos_fsm(pm, site, paxos_ack_accept, p);
6103 }
6104 }
6105
6106 236870 static void process_learn_op(site_def const *site, pax_msg *p,
6107 linkage *reply_queue) {
6108 236870 pax_machine *pm = get_cache(p->synode);
6109
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 236870 times.
 236870 assert(pm);
6110 (void)reply_queue;
6111
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 236870 times.
 236870 if (p->force_delivery) pm->force_delivery = 1;
6112 236870 update_max_synode(p);
6113 236870 paxos_fsm(pm, site, paxos_learn, p);
6114 236870 handle_learn(site, pm, p);
6115 236870 }
6116
6117 45731 static void process_recover_learn_op(site_def const *site, pax_msg *p,
6118 linkage *reply_queue) {
6119 45731 pax_machine *pm = get_cache(p->synode);
6120
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 45731 times.
 45731 assert(pm);
6121 (void)reply_queue;
6122 IFDBG(D_NONE, FN; STRLIT("recover_learn_op receive "); SYCEXP(p->synode));
6123
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 45731 times.
 45731 if (p->force_delivery) pm->force_delivery = 1;
6124 45731 update_max_synode(p);
6125 {
6126 IFDBG(D_NONE, FN; STRLIT("recover_learn_op learn "); SYCEXP(p->synode));
6127 45731 p->op = learn_op;
6128 45731 paxos_fsm(pm, site, paxos_learn, p);
6129 45731 handle_learn(site, pm, p);
6130 }
6131 45731 }
6132
6133 256248 static void process_skip_op(site_def const *site, pax_msg *p,
6134 linkage *reply_queue) {
6135 (void)reply_queue;
6136 256248 pax_machine *pm = get_cache(p->synode);
6137
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 256248 times.
 256248 assert(pm);
6138
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 256248 times.
 256248 if (p->force_delivery) pm->force_delivery = 1;
6139 256248 paxos_fsm(pm, site, paxos_learn, p);
6140 256248 handle_skip(site, pm, p);
6141 256248 }
6142
6143 154336 static void process_i_am_alive_op(site_def const *site, pax_msg *p,
6144 linkage *reply_queue) {
6145 /* Update max_synode, but use only p->max_synode, ignore p->synode */
6146
1/2
✓ Branch 0 taken 154336 times.
✗ Branch 1 not taken.
 154336 if (!is_dead_site(p->group_id)) {
6147
4/4
✓ Branch 0 taken 154069 times.
✓ Branch 1 taken 267 times.
✓ Branch 2 taken 2321 times.
✓ Branch 3 taken 152015 times.
 308405 if (max_synode.group_id == p->synode.group_id &&
6148
2/2
✓ Branch 0 taken 2321 times.
✓ Branch 1 taken 151748 times.
 154069 synode_gt(p->max_synode, max_synode)) {
6149 2321 set_max_synode(p->max_synode);
6150 }
6151 }
6152 154336 handle_alive(site, reply_queue, p);
6153 154336 }
6154
6155 5914 static void process_are_you_alive_op(site_def const *site, pax_msg *p,
6156 linkage *reply_queue) {
6157 5914 handle_alive(site, reply_queue, p);
6158 5914 }
6159
6160 ✗ static void process_need_boot_op(site_def const *site, pax_msg *p,
6161 linkage *reply_queue) {
6162 /* purecov: begin deadcode */
6163 /* Only in run state. Test state and do it here because we need to use
6164 * reply queue */
6165 ✗ if (can_send_snapshot() &&
6166 ✗ !synode_eq(get_site_def()->boot_key, null_synode)) {
6167 ✗ handle_boot(site, reply_queue, p);
6168 }
6169 /* Wake senders waiting to connect, since new node has appeared */
6170 ✗ wakeup_sender();
6171 /* purecov: end */
6172 }
6173
6174 ✗ static void process_die_op(site_def const *site, pax_msg *p,
6175 linkage *reply_queue) {
6176 (void)reply_queue;
6177 /* assert("die horribly" == "need coredump"); */
6178 {
6179 ✗ GET_GOUT;
6180 ✗ FN;
6181 ✗ STRLIT("die_op ");
6182 ✗ SYCEXP(executed_msg);
6183 ✗ SYCEXP(delivered_msg);
6184 ✗ SYCEXP(p->synode);
6185 ✗ SYCEXP(p->delivered_msg);
6186 ✗ SYCEXP(p->max_synode);
6187 ✗ PRINT_GOUT;
6188 ✗ FREE_GOUT;
6189 }
6190 /*
6191 If the message with the number in the incoming die_op message
6192 already has been executed (delivered), then it means that we
6193 actually got consensus on it, since otherwise we would not have
6194 delivered it.Such a situation could arise if one of the nodes has
6195 expelled the message from its cache, but others have not. So when
6196 sending out a request, we might get two different answers, one
6197 indicating that we are too far behind and should restart, and
6198 another with the actual consensus value. If the value arrives
6199 first, we will deliver it, and then the die_op may arrive later.
6200 But it this case it does not matter, since we got what we needed
6201 anyway. It is only a partial guard against exiting without really
6202 needing it of course, since the die_op may arrive first, and we
6203 do not wait for a die_op from all the other nodes. We could do
6204 that with some extra housekeeping in the pax_machine (a new bit
6205 vector), but I am not convinced that it is worth the effort.
6206 */
6207 ✗ if (!synode_lt(p->synode, executed_msg)) {
6208 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("terminating"));)
6209 ✗ g_critical("Node %u is unable to get message {%x %" PRIu64
6210 " %u}, since the group is too far "
6211 "ahead. Node will now exit.",
6212 get_nodeno(site), SY_MEM(p->synode));
6213 ✗ terminate_and_exit();
6214 }
6215 }
6216
6217 333399 static void process_read_op(site_def const *site, pax_msg *p,
6218 linkage *reply_queue) {
6219 333399 pax_machine *pm = get_cache(p->synode);
6220
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 333399 times.
 333399 assert(pm);
6221
6222 333399 handle_read(site, pm, reply_queue, p);
6223 333399 }
6224
6225 1360 static void process_gcs_snapshot_op(site_def const *site, pax_msg *p,
6226 linkage *reply_queue) {
6227 (void)site;
6228 (void)reply_queue;
6229 /* Avoid duplicate snapshots and snapshots from zombies */
6230 IFDBG(D_BASE, FN; SYCEXP(executed_msg););
6231 IFDBG(D_BASE, FN; SYCEXP(start_config););
6232
3/6
✓ Branch 0 taken 1360 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1360 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1360 times.
✗ Branch 5 not taken.
 2720 if (!synode_eq(start_config, get_highest_boot_key(p->gcs_snap)) &&
6233 1360 !is_dead_site(p->group_id)) {
6234 1360 update_max_synode(p);
6235 /* For incoming messages, note delivery of snapshot from sender node */
6236 1360 note_snapshot(p->from);
6237
3/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1360 times.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 1359 times.
 1360 XCOM_FSM(x_fsm_snapshot, void_arg(p->gcs_snap));
6238 }
6239 1360 }
6240
6241 240065 static void process_tiny_learn_op(site_def const *site, pax_msg *p,
6242 linkage *reply_queue) {
6243
2/2
✓ Branch 0 taken 24434 times.
✓ Branch 1 taken 215631 times.
 240065 if (p->msg_type == no_op) {
6244 24434 process_learn_op(site, p, reply_queue);
6245 } else {
6246 215631 pax_machine *pm = get_cache(p->synode);
6247
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 215631 times.
 215631 assert(pm);
6248
2/2
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 215551 times.
 215631 if (p->force_delivery) pm->force_delivery = 1;
6249 215631 handle_tiny_learn(site, pm, p);
6250 }
6251 240065 }
6252
6253 /* If this node is leader, grant a synode number for use by secondary.
6254 Send reply as synode_allocated. */
6255 461 static void process_synode_request(site_def const *site, pax_msg *p,
6256 linkage *reply_queue) {
6257 (void)site;
6258
6259 /* Find a free slot */
6260
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 461 times.
 461 assert(!synode_eq(current_message, null_synode));
6261 IFDBG(D_CONS, FN; SYCEXP(executed_msg); SYCEXP(current_message));
6262 461 site_def *tmp_site = find_site_def_rw(current_message);
6263 /* See if we can do anything with this message */
6264
6/8
✓ Branch 0 taken 461 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 461 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 456 times.
✓ Branch 5 taken 5 times.
✓ Branch 6 taken 456 times.
✓ Branch 7 taken 5 times.
 461 if (tmp_site && get_nodeno(tmp_site) != VOID_NODE_NO && is_leader(tmp_site)) {
6265 /* Send reply with msgno */
6266
1/2
✓ Branch 0 taken 456 times.
✗ Branch 1 not taken.
 456 synode_no msgno = local_synode_allocator(current_message);
6267 /* Ensure that reply is sane. Note that we only allocate `msgno` *if* next
6268 synod is still within the event horizon. This effectively means that
6269 the leader always reserves at least one synod to himself, the last
6270 synod of the event horizon. The point is to ensure that the leader does
6271 not allocate all the possible synods to a non-leader that then doesn't
6272 act on them, e.g. by crashing. The last synod of the event horizon will
6273 always be up for grabs either by the leader, or if the leader is
6274 suspected, by some non-leader that will self-allocate that synod to
6275 himself. */
6276
6/8
✓ Branch 0 taken 456 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 456 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 440 times.
✓ Branch 5 taken 16 times.
✓ Branch 6 taken 440 times.
✓ Branch 7 taken 16 times.
 896 if (!(too_far(incr_msgno(msgno)) ||
6277
3/6
✓ Branch 0 taken 440 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 440 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 440 times.
✗ Branch 5 not taken.
 440 ignore_message(msgno, find_site_def_rw(msgno),
6278 "process_synode_request"))) {
6279 // We will grab this number, advance current_message
6280
2/4
✓ Branch 0 taken 440 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 440 times.
✗ Branch 3 not taken.
 440 set_current_message(incr_synode(msgno));
6281 IFDBG(D_CONS, FN; STRLIT("sending reply "); SYCEXP(executed_msg);
6282 SYCEXP(current_message); SYCEXP(msgno));
6283
2/4
✓ Branch 0 taken 440 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 440 times.
✗ Branch 3 not taken.
 440 CREATE_REPLY(p);
6284 440 reply->synode = msgno;
6285 440 reply->op = synode_allocated;
6286 IFDBG(D_CONS, FN; SYCEXP(msgno));
6287
4/10
✓ Branch 0 taken 440 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 440 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 440 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 440 times.
✗ Branch 9 not taken.
 440 SEND_REPLY;
6288 } else {
6289 IFDBG(D_CONS, FN; STRLIT("not sending reply "); SYCEXP(executed_msg);
6290 SYCEXP(msgno));
6291 }
6292 } else {
6293 IFDBG(D_CONS, FN; STRLIT("not leader ");
6294 if (tmp_site) SYCEXP(tmp_site->start));
6295 }
6296 461 }
6297
6298 /* If this node is secondary, add synode to set of available synodes */
6299 456 static void process_synode_allocated(site_def const *site, pax_msg *p,
6300 linkage *reply_queue) {
6301 (void)site;
6302 (void)p;
6303 (void)reply_queue;
6304
6305 IFDBG(D_BASE, FN; SYCEXP(p->synode));
6306 456 synode_number_pool.put(p->synode, synode_allocation_type::remote);
6307 456 }
6308
6309 static msg_handler dispatch_table[LAST_OP] = {process_client_msg,
6310 nullptr,
6311 process_prepare_op,
6312 process_ack_prepare_op,
6313 process_ack_prepare_op,
6314 process_accept_op,
6315 process_ack_accept_op,
6316 process_learn_op,
6317 process_recover_learn_op,
6318 nullptr,
6319 nullptr,
6320 nullptr,
6321 nullptr,
6322 nullptr,
6323 process_skip_op,
6324 process_i_am_alive_op,
6325 process_are_you_alive_op,
6326 process_need_boot_op,
6327 nullptr,
6328 process_die_op,
6329 process_read_op,
6330 process_gcs_snapshot_op,
6331 nullptr,
6332 process_tiny_learn_op,
6333 process_synode_request,
6334 process_synode_allocated};
6335
6336 68 static msg_handler *clone_dispatch_table(msg_handler const *proto) {
6337 msg_handler *clone =
6338 68 static_cast<msg_handler *>(xcom_calloc(1, sizeof(dispatch_table)));
6339
1/2
✓ Branch 0 taken 68 times.
✗ Branch 1 not taken.
 68 if (clone)
6340 68 memcpy(clone, proto, sizeof(dispatch_table));
6341 else
6342 ✗ oom_abort = 1;
6343 68 return clone;
6344 }
6345
6346 33 static msg_handler *primary_dispatch_table() {
6347 33 msg_handler *clone = clone_dispatch_table(dispatch_table);
6348 33 return clone;
6349 }
6350
6351 35 static msg_handler *secondary_dispatch_table() {
6352 35 msg_handler *clone = clone_dispatch_table(dispatch_table);
6353
1/2
✓ Branch 0 taken 35 times.
✗ Branch 1 not taken.
 35 if (clone) {
6354 35 clone[synode_request] = nullptr;
6355 }
6356 35 return clone;
6357 }
6358
6359 1925392 pax_msg *dispatch_op(site_def const *site, pax_msg *p, linkage *reply_queue) {
6360 1925392 site_def *dsite = find_site_def_rw(p->synode);
6361
6362
8/8
✓ Branch 0 taken 1817352 times.
✓ Branch 1 taken 108040 times.
✓ Branch 2 taken 1753002 times.
✓ Branch 3 taken 64350 times.
✓ Branch 4 taken 1549272 times.
✓ Branch 5 taken 203730 times.
✓ Branch 6 taken 1549272 times.
✓ Branch 7 taken 376120 times.
 1925392 if (dsite && p->op != client_msg && is_server_connected(dsite, p->from)) {
6363 /* Wake up the detector task if this node was previously marked as
6364 * potentially failed. */
6365
2/2
✓ Branch 0 taken 43225 times.
✓ Branch 1 taken 1506047 times.
 1549272 if (!note_detected(dsite, p->from)) task_wakeup(&detector_wait);
6366 1549272 update_delivered(dsite, p->from, p->delivered_msg);
6367 }
6368
6369 IFDBG(D_BASE, FN; STRLIT("incoming message ");
6370 COPY_AND_FREE_GOUT(dbg_pax_msg(p)););
6371 ADD_DBG(D_DISPATCH, add_synode_event(p->synode);
6372 add_event(EVENT_DUMP_PAD, string_arg("p->from"));
6373 add_event(EVENT_DUMP_PAD, uint_arg(p->from));
6374 add_event(EVENT_DUMP_PAD, string_arg("too_far(p->synode)"));
6375 add_event(EVENT_DUMP_PAD, int_arg(too_far(p->synode)));
6376 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(p->op))););
6377
6378
2/4
✓ Branch 0 taken 1925392 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1925392 times.
✗ Branch 3 not taken.
 1925392 if (p->op >= 0 && p->op < LAST_OP) {
6379
4/4
✓ Branch 0 taken 1753002 times.
✓ Branch 1 taken 172390 times.
✓ Branch 2 taken 12249 times.
✓ Branch 3 taken 1740753 times.
 1925392 if (site && site->dispatch_table) { /* Use site-specific dispatch if any */
6380
1/2
✓ Branch 0 taken 12249 times.
✗ Branch 1 not taken.
 12249 if (site->dispatch_table[p->op])
6381 12249 site->dispatch_table[p->op](site, p, reply_queue);
6382 } else {
6383
1/2
✓ Branch 0 taken 1913143 times.
✗ Branch 1 not taken.
 1913143 if (dispatch_table[p->op]) dispatch_table[p->op](site, p, reply_queue);
6384 }
6385 } else {
6386 ✗ G_WARNING("No possible handler for message %d %s", p->op,
6387 pax_op_to_str(p->op));
6388 }
6389
6390
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1925392 times.
 1925392 if (oom_abort) {
6391 ✗ g_critical("Node %u has run out of memory and will now exit.",
6392 get_nodeno(site));
6393 ✗ terminate_and_exit();
6394 }
6395 1925392 return (p);
6396 }
6397
6398 /* Acceptor-learner task */
6399 #define SERIALIZE_REPLY(msg) \
6400 msg->to = ep->p->from; \
6401 msg->from = ep->p->to; \
6402 msg->delivered_msg = get_delivered_msg(); \
6403 msg->max_synode = get_max_synode(); \
6404 serialize_msg(msg, ep->rfd->x_proto, &ep->buflen, &ep->buf);
6405
6406 #define WRITE_REPLY \
6407 if (ep->buflen) { \
6408 int64_t sent; \
6409 IFDBG(D_TRANSPORT, FN; STRLIT("task_write "); NDBG(ep->rfd.fd, d); \
6410 NDBG(ep->buflen, u)); \
6411 TASK_CALL(task_write(ep->rfd, ep->buf, ep->buflen, &sent)); \
6412 send_count[ep->p->op]++; \
6413 send_bytes[ep->p->op] += ep->buflen; \
6414 X_FREE(ep->buf); \
6415 } \
6416 ep->buf = NULL;
6417
6418 982222 static inline void update_srv(server **target, server *srv) {
6419
2/2
✓ Branch 0 taken 712963 times.
✓ Branch 1 taken 269259 times.
 982222 if (srv) srv_ref(srv);
6420
2/2
✓ Branch 0 taken 712952 times.
✓ Branch 1 taken 269270 times.
 982222 if (*target) srv_unref(*target);
6421 982222 *target = srv;
6422 982222 }
6423
6424 /* A message is harmless if it cannot change the outcome of a consensus round.
6425 * learn_op does change the value, but we trust that the sender has correctly
6426 * derived the value from a majority of the acceptors, so in that sense it is
6427 * harmless. */
6428 973405 static int harmless(pax_msg const *p) {
6429
2/2
✓ Branch 0 taken 1792 times.
✓ Branch 1 taken 971613 times.
 973405 if (p->synode.msgno == 0) return 1;
6430
2/2
✓ Branch 0 taken 343926 times.
✓ Branch 1 taken 627687 times.
 971613 switch (p->op) {
6431 343926 case i_am_alive_op:
6432 case are_you_alive_op:
6433 case need_boot_op:
6434 case gcs_snapshot_op:
6435 case learn_op:
6436 case recover_learn_op:
6437 case tiny_learn_op:
6438 case die_op:
6439 343926 return 1;
6440 627687 default:
6441 627687 return 0;
6442 }
6443 }
6444
6445 1033404 static int wait_for_cache(pax_machine **pm, synode_no synode, double timeout) {
6446 DECL_ENV
6447 double now;
6448 1033404 ENV_INIT
6449 1033404 END_ENV_INIT
6450 END_ENV;
6451
6452
4/9
✓ Branch 0 taken 1033404 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 1033404 times.
✓ Branch 5 taken 1033404 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 1033404 times.
 1033404 TASK_BEGIN
6453 1033404 ep->now = task_now();
6454
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1033404 times.
 1033404 while ((*pm = get_cache(synode)) == nullptr) {
6455 /* Wait for executor to make progress */
6456 ✗ TIMED_TASK_WAIT(&exec_wait, 0.5);
6457 ✗ if (task_now() - ep->now > timeout) break; /* Timeout, return NULL. */
6458 }
6459 1033404 FINALLY
6460
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1033404 times.
 1033404 TASK_END;
6461 }
6462
6463 /*
6464 Verify if we need to poll the cache before calling dispatch_op.
6465 Avoid waiting for a machine if it is not going to be used.
6466 */
6467 926654 static bool_t should_poll_cache(pax_op op) {
6468
6/8
✓ Branch 0 taken 926654 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 925294 times.
✓ Branch 3 taken 1360 times.
✓ Branch 4 taken 925294 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 1792 times.
✓ Branch 7 taken 923502 times.
 926654 if (op == die_op || op == gcs_snapshot_op || op == initial_op ||
6469 op == client_msg)
6470 3152 return FALSE;
6471 923502 return TRUE;
6472 }
6473
6474 422720 int acceptor_learner_task(task_arg arg) {
6475 DECL_ENV
6476 connection_descriptor *rfd;
6477 srv_buf *in_buf;
6478 pax_msg *p;
6479 u_int buflen;
6480 char *buf;
6481 linkage reply_queue;
6482 int errors;
6483 server *srv;
6484 site_def const *site;
6485 int behind;
6486 7752 ENV_INIT
6487 7752 END_ENV_INIT
6488 END_ENV;
6489
6490 422720 int64_t n{0};
6491 422720 pax_machine *pm{nullptr};
6492
6493
8/21
✓ Branch 0 taken 7752 times.
✓ Branch 1 taken 7752 times.
✓ Branch 2 taken 407216 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 7752 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 7752 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✓ Branch 16 taken 7752 times.
✓ Branch 17 taken 7752 times.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
✓ Branch 20 taken 7752 times.
 422720 TASK_BEGIN
6494
6495 7752 ep->rfd = (connection_descriptor *)get_void_arg(arg);
6496 7752 ep->in_buf = (srv_buf *)xcom_calloc(1, sizeof(srv_buf));
6497 7752 ep->p = nullptr;
6498 7752 ep->buflen = 0;
6499 7752 ep->buf = nullptr;
6500 7752 ep->errors = 0;
6501 7752 ep->srv = nullptr;
6502 7752 ep->behind = FALSE;
6503
6504 /* We have a connection, make socket non-blocking and wait for request */
6505
1/2
✓ Branch 0 taken 7752 times.
✗ Branch 1 not taken.
 7752 unblock_fd(ep->rfd->fd);
6506
1/2
✓ Branch 0 taken 7752 times.
✗ Branch 1 not taken.
 7752 set_nodelay(ep->rfd->fd);
6507
1/2
✓ Branch 0 taken 7752 times.
✗ Branch 1 not taken.
 7752 wait_io(stack, ep->rfd->fd, 'r');
6508
3/6
✗ Branch 0 not taken.
✓ Branch 1 taken 7752 times.
✓ Branch 2 taken 7752 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 7752 times.
 15504 TASK_YIELD;
6509
6510 7752 set_connected(ep->rfd, CON_FD);
6511 7752 link_init(&ep->reply_queue, TYPE_HASH("msg_link"));
6512
6513 8828 again:
6514
1/2
✓ Branch 0 taken 982233 times.
✗ Branch 1 not taken.
 982233 while (!xcom_shutdown) {
6515 982233 ep->site = nullptr;
6516
2/4
✓ Branch 0 taken 982233 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 982233 times.
✗ Branch 3 not taken.
 982233 unchecked_replace_pax_msg(&ep->p, pax_msg_new_0(null_synode));
6517
6518
1/2
✓ Branch 0 taken 982233 times.
✗ Branch 1 not taken.
 982233 if (use_buffered_read) {
6519
12/18
✓ Branch 0 taken 982233 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1389449 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 982222 times.
✓ Branch 5 taken 407227 times.
✓ Branch 6 taken 1700 times.
✓ Branch 7 taken 980522 times.
✓ Branch 8 taken 407227 times.
✓ Branch 9 taken 980522 times.
✗ Branch 10 not taken.
✓ Branch 11 taken 407216 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 407216 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✓ Branch 16 taken 407216 times.
✓ Branch 17 taken 980522 times.
 1796665 TASK_CALL(buffered_read_msg(ep->rfd, ep->in_buf, ep->p, ep->srv, &n));
6520 } else {
6521 /* purecov: begin deadcode */
6522 ✗ TASK_CALL(read_msg(ep->rfd, ep->p, ep->srv, &n));
6523 /* purecov: end */
6524 }
6525 ADD_DBG(D_NONE, add_synode_event(ep->p->synode);
6526 add_event(EVENT_DUMP_PAD, string_arg("ep->p->from"));
6527 add_event(EVENT_DUMP_PAD, uint_arg(ep->p->from));
6528 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(ep->p->op))););
6529
6530
8/10
✓ Branch 0 taken 711794 times.
✓ Branch 1 taken 268728 times.
✓ Branch 2 taken 711794 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 711794 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 657232 times.
✓ Branch 7 taken 54562 times.
✓ Branch 8 taken 657232 times.
✓ Branch 9 taken 323290 times.
 1692316 if (ep->srv && !ep->srv->invalid && ((int)ep->p->op != (int)client_msg) &&
6531 711794 is_connected(ep->srv->con))
6532
1/2
✓ Branch 0 taken 657232 times.
✗ Branch 1 not taken.
 657232 server_detected(ep->srv);
6533
6534
2/4
✓ Branch 0 taken 980522 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 980522 times.
 980522 if (((int)ep->p->op < (int)client_msg || ep->p->op > LAST_OP)) {
6535 /* invalid operation, ignore message */
6536 ✗ delete_pax_msg(ep->p);
6537 ✗ ep->p = nullptr;
6538 ✗ TASK_YIELD;
6539 ✗ continue;
6540 }
6541
2/2
✓ Branch 0 taken 6041 times.
✓ Branch 1 taken 974481 times.
 980522 if (n <= 0) {
6542 6041 break;
6543 }
6544
2/2
✓ Branch 0 taken 972689 times.
✓ Branch 1 taken 1792 times.
 974481 if (ep->p->op != client_msg) { // Clients have no site
6545
1/2
✓ Branch 0 taken 972689 times.
✗ Branch 1 not taken.
 972689 ep->site = find_site_def(ep->p->synode);
6546 }
6547
6548 /* Handle this connection on a local_server task instead of this
6549 acceptor_learner_task task. */
6550 /* purecov: begin deadcode */
6551
3/4
✓ Branch 0 taken 1792 times.
✓ Branch 1 taken 972689 times.
✓ Branch 2 taken 1792 times.
✗ Branch 3 not taken.
 974481 if (ep->p->op == client_msg && ep->p->a &&
6552
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1792 times.
 1792 ep->p->a->body.c_t == convert_into_local_server_type) {
6553 ✗ if (local_server_is_setup()) {
6554 /* Launch local_server task to handle this connection. */
6555 {
6556 ✗ connection_descriptor *con = (connection_descriptor *)xcom_malloc(
6557 sizeof(connection_descriptor));
6558 ✗ *con = *ep->rfd;
6559 ✗ task_new(local_server, void_arg(con), "local_server",
6560 XCOM_THREAD_DEBUG);
6561 }
6562 }
6563 /* Reply to client:
6564 - OK if local_server task is setup, or
6565 - FAIL otherwise. */
6566 {
6567 ✗ CREATE_REPLY(ep->p);
6568 ✗ reply->op = xcom_client_reply;
6569 ✗ reply->cli_err = local_server_is_setup() ? REQUEST_OK : REQUEST_FAIL;
6570 ✗ SERIALIZE_REPLY(reply);
6571 ✗ replace_pax_msg(&reply, nullptr);
6572 }
6573 ✗ WRITE_REPLY;
6574 ✗ delete_pax_msg(ep->p);
6575 ✗ ep->p = nullptr;
6576 ✗ if (local_server_is_setup()) {
6577 /* Relinquish ownership of the connection. It is now onwed by the
6578 launched local_server task. */
6579 ✗ reset_connection(ep->rfd);
6580 }
6581 /* Terminate this task. */
6582 ✗ TERMINATE;
6583 }
6584 /* purecov: end */
6585
6586 /*
6587 Getting a pointer to the server needs to be done after we have
6588 received a message, since without having received a message, we
6589 cannot know who it is from. We could peek at the message and de‐
6590 serialize the message number and from field, but since the server
6591 does not change, it should be sufficient to cache the server in
6592 the acceptor_learner task. A cleaner solution would have been to
6593 move the timestamps out of the server object, and have a map in‐
6594 dexed by IP/port or UUID to track the timestamps, since this is
6595 common to both the sender_task, reply_handler_task, and the ac‐
6596 ceptor_learner_task.
6597 */
6598
1/2
✓ Branch 0 taken 974481 times.
✗ Branch 1 not taken.
 974481 update_srv(&ep->srv, get_server(ep->site, ep->p->from));
6599 974481 ep->p->refcnt = 1; /* Refcnt from other end is void here */
6600 IFDBG(D_NONE, FN; NDBG(ep->rfd.fd, d); NDBG(task_now(), f);
6601 COPY_AND_FREE_GOUT(dbg_pax_msg(ep->p)););
6602 974481 receive_count[ep->p->op]++;
6603 974481 receive_bytes[ep->p->op] += (uint64_t)n + MSG_HDR_SIZE;
6604 {
6605
3/4
✓ Branch 0 taken 974481 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 908451 times.
✓ Branch 3 taken 66030 times.
 974481 if (get_maxnodes(ep->site) > 0) {
6606 908451 ep->behind = ep->p->synode.msgno < delivered_msg.msgno;
6607 }
6608 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("before dispatch "));
6609 add_synode_event(ep->p->synode);
6610 add_event(EVENT_DUMP_PAD, string_arg("ep->p->from"));
6611 add_event(EVENT_DUMP_PAD, uint_arg(ep->p->from));
6612 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(ep->p->op)));
6613 add_event(EVENT_DUMP_PAD,
6614 string_arg(pax_msg_type_to_str(ep->p->msg_type)));
6615 add_event(EVENT_DUMP_PAD, string_arg("is_cached(ep->p->synode)"));
6616 add_event(EVENT_DUMP_PAD, int_arg(is_cached(ep->p->synode)));
6617 add_event(EVENT_DUMP_PAD, string_arg("behind"));
6618 add_event(EVENT_DUMP_PAD, int_arg(ep->behind)););
6619 /* Special treatment to see if synode number is valid. Return no-op if
6620 * not. */
6621
4/4
✓ Branch 0 taken 640006 times.
✓ Branch 1 taken 334475 times.
✓ Branch 2 taken 612309 times.
✓ Branch 3 taken 27697 times.
 974481 if (ep->p->op == read_op || ep->p->op == prepare_op ||
6622
2/2
✓ Branch 0 taken 122399 times.
✓ Branch 1 taken 489910 times.
 612309 ep->p->op == accept_op) {
6623
2/2
✓ Branch 0 taken 448087 times.
✓ Branch 1 taken 36484 times.
 484571 if (ep->site) {
6624 ADD_DBG(
6625 D_BASE, add_event(EVENT_DUMP_PAD, string_arg("ep->p->synode"));
6626 add_synode_event(ep->p->synode);
6627 add_event(EVENT_DUMP_PAD, string_arg("ep->site->start"));
6628 add_synode_event(ep->site->start); add_event(
6629 EVENT_DUMP_PAD, string_arg("ep->site->nodes.node_list_len"));
6630 add_event(EVENT_DUMP_PAD,
6631 uint_arg(ep->site->nodes.node_list_len)););
6632
2/2
✓ Branch 0 taken 1076 times.
✓ Branch 1 taken 447011 times.
 448087 if (ep->p->synode.node >= ep->site->nodes.node_list_len) {
6633 {
6634
2/4
✓ Branch 0 taken 1076 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1076 times.
✗ Branch 3 not taken.
 1076 CREATE_REPLY(ep->p);
6635
1/2
✓ Branch 0 taken 1076 times.
✗ Branch 1 not taken.
 1076 create_noop(reply);
6636 1076 set_learn_type(reply);
6637
3/6
✓ Branch 0 taken 1076 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1076 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1076 times.
✗ Branch 5 not taken.
 1076 SERIALIZE_REPLY(reply);
6638
1/2
✓ Branch 0 taken 1076 times.
✗ Branch 1 not taken.
 1076 delete_pax_msg(reply); /* Deallocate BEFORE potentially blocking
6639 call which will lose value of reply */
6640 }
6641
7/20
✓ Branch 0 taken 1076 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1076 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1076 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 1076 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 1076 times.
✗ Branch 10 not taken.
✓ Branch 11 taken 1076 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
✓ Branch 19 taken 1076 times.
 1076 WRITE_REPLY;
6642 1076 goto again;
6643 }
6644 }
6645 }
6646 /* Reject any message that might compromise the integrity of a consensus
6647 * instance. We do this by not processing any message which may change
6648 * the
6649 * outcome if the consensus instance has been evicted from the cache */
6650 973405 if (harmless(ep->p) || /* Harmless message */
6651
7/8
✓ Branch 0 taken 627687 times.
✓ Branch 1 taken 345718 times.
✓ Branch 2 taken 627687 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 158688 times.
✓ Branch 5 taken 468999 times.
✓ Branch 6 taken 926654 times.
✓ Branch 7 taken 46751 times.
 1132093 is_cached(ep->p->synode) || /* Already in cache */
6652
2/2
✓ Branch 0 taken 111937 times.
✓ Branch 1 taken 46751 times.
 158688 (!ep->behind)) { /* Guard against cache pollution from other nodes
6653 */
6654
6655
2/2
✓ Branch 0 taken 923502 times.
✓ Branch 1 taken 3152 times.
 926654 if (should_poll_cache(ep->p->op)) {
6656
6/18
✓ Branch 0 taken 923502 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 923502 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 923502 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 923502 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 923502 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 923502 times.
 923502 TASK_CALL(wait_for_cache(&pm, ep->p->synode, 10));
6657
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 923502 times.
 923502 if (!pm) continue; /* Could not get a machine, discarding message. */
6658 }
6659
6660
1/2
✓ Branch 0 taken 926654 times.
✗ Branch 1 not taken.
 926654 dispatch_op(ep->site, ep->p, &ep->reply_queue);
6661
6662 /* Send replies on same fd */
6663
2/2
✓ Branch 0 taken 247561 times.
✓ Branch 1 taken 926654 times.
 1174215 while (!link_empty(&ep->reply_queue)) {
6664 {
6665 msg_link *reply =
6666
1/2
✓ Branch 0 taken 247561 times.
✗ Branch 1 not taken.
 247561 (msg_link *)(link_extract_first(&ep->reply_queue));
6667 IFDBG(D_DISPATCH, FN; PTREXP(reply);
6668 COPY_AND_FREE_GOUT(dbg_linkage(&ep->reply_queue));
6669 COPY_AND_FREE_GOUT(dbg_msg_link(reply));
6670 COPY_AND_FREE_GOUT(dbg_pax_msg(reply->p)););
6671
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 247561 times.
 247561 assert(reply->p);
6672
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 247561 times.
 247561 assert(reply->p->refcnt > 0);
6673 IFDBG(D_DISPATCH, FN; STRLIT("serialize "); PTREXP(reply));
6674
3/6
✓ Branch 0 taken 247561 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 247561 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 247561 times.
✗ Branch 5 not taken.
 247561 SERIALIZE_REPLY(reply->p);
6675
1/2
✓ Branch 0 taken 247561 times.
✗ Branch 1 not taken.
 247561 msg_link_delete(&reply); /* Deallocate BEFORE potentially blocking
6676 call which will lose value of reply */
6677 }
6678
7/20
✓ Branch 0 taken 247561 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 247561 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 247561 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 247561 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 247561 times.
✗ Branch 10 not taken.
✓ Branch 11 taken 247561 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
✓ Branch 19 taken 247561 times.
 247561 WRITE_REPLY;
6679 }
6680 } else {
6681 IFDBG(D_EXEC, FN; STRLIT("rejecting ");
6682 STRLIT(pax_op_to_str(ep->p->op)); NDBG(ep->p->from, d);
6683 NDBG(ep->p->to, d); SYCEXP(ep->p->synode);
6684 BALCEXP(ep->p->proposal));
6685
1/2
✓ Branch 0 taken 46751 times.
✗ Branch 1 not taken.
 46751 if (/* xcom_booted() && */ ep->behind) {
6686
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46751 times.
 46751 if (/*ep->p->op == prepare_op && */ was_removed_from_cache(
6687
1/2
✓ Branch 0 taken 46751 times.
✗ Branch 1 not taken.
 46751 ep->p->synode)) {
6688 IFDBG(D_NONE, FN; STRLIT("send_die ");
6689 STRLIT(pax_op_to_str(ep->p->op)); NDBG(ep->p->from, d);
6690 NDBG(ep->p->to, d); SYCEXP(ep->p->synode);
6691 BALCEXP(ep->p->proposal));
6692 ✗ if (get_maxnodes(ep->site) > 0) {
6693 {
6694 ✗ pax_msg *np = nullptr;
6695 ✗ np = pax_msg_new(ep->p->synode, ep->site);
6696 ✗ np->op = die_op;
6697 ✗ SERIALIZE_REPLY(np);
6698 IFDBG(D_NONE, FN; STRLIT("sending die_op to node ");
6699 NDBG(np->to, d); SYCEXP(executed_msg); SYCEXP(max_synode);
6700 SYCEXP(np->synode));
6701 ✗ delete_pax_msg(np); /* Deallocate BEFORE potentially blocking
6702 call which will lose value of np */
6703 }
6704 ✗ WRITE_REPLY;
6705 }
6706 }
6707 }
6708 }
6709 }
6710 /* TASK_YIELD; */
6711 }
6712
6713 ✗ FINALLY
6714 IFDBG(D_BUG, FN; STRLIT(" shutdown "); NDBG(ep->rfd.fd, d);
6715 NDBG(task_now(), f));
6716
3/6
✓ Branch 0 taken 7741 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 7741 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 7741 times.
 7741 if (ep->reply_queue.suc && !link_empty(&ep->reply_queue))
6717 ✗ empty_msg_list(&ep->reply_queue);
6718
1/2
✓ Branch 0 taken 7741 times.
✗ Branch 1 not taken.
 7741 unchecked_replace_pax_msg(&ep->p, nullptr);
6719
1/2
✓ Branch 0 taken 7741 times.
✗ Branch 1 not taken.
 7741 shutdown_connection(ep->rfd);
6720 7741 free(ep->rfd);
6721 IFDBG(D_NONE, FN; NDBG(xcom_shutdown, d));
6722
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7741 times.
 7741 if (ep->buf) X_FREE(ep->buf);
6723 7741 free(ep->in_buf);
6724
6725 /* Unref srv to avoid leak */
6726
1/2
✓ Branch 0 taken 7741 times.
✗ Branch 1 not taken.
 7741 update_srv(&ep->srv, nullptr);
6727
6728 IFDBG(D_BUG, FN; STRLIT(" shutdown completed"); NDBG(ep->rfd.fd, d);
6729 NDBG(task_now(), f));
6730
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 7741 times.
✓ Branch 2 taken 7741 times.
✗ Branch 3 not taken.
 7741 TASK_END;
6731 }
6732
6733 /* Reply handler task */
6734
6735 static void server_handle_need_snapshot(server *srv, site_def const *s,
6736 node_no node);
6737
6738 393390 int reply_handler_task(task_arg arg) {
6739 DECL_ENV
6740 server *s;
6741 pax_msg *reply;
6742 double dtime;
6743 2992 ENV_INIT
6744 2992 END_ENV_INIT
6745 END_ENV;
6746
6747 393390 int64_t n{0};
6748
9/15
✓ Branch 0 taken 2992 times.
✓ Branch 1 taken 10697 times.
✓ Branch 2 taken 132558 times.
✓ Branch 3 taken 247143 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 2992 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 2992 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 2992 times.
✓ Branch 11 taken 2992 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✓ Branch 14 taken 2992 times.
 393390 TASK_BEGIN
6749
6750 2992 ep->dtime = INITIAL_CONNECT_WAIT; /* Initial wait is short, to avoid
6751 unnecessary waiting */
6752 2992 ep->s = (server *)get_void_arg(arg);
6753
1/2
✓ Branch 0 taken 2992 times.
✗ Branch 1 not taken.
 2992 srv_ref(ep->s);
6754 2992 ep->reply = nullptr;
6755
6756
1/2
✓ Branch 0 taken 252115 times.
✗ Branch 1 not taken.
 252115 while (!xcom_shutdown) {
6757
2/2
✓ Branch 0 taken 10697 times.
✓ Branch 1 taken 250828 times.
 261525 while (!is_connected(ep->s->con)) {
6758 IFDBG(D_NONE, FN; STRLIT("waiting for connection"));
6759
6/10
✓ Branch 0 taken 10697 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 10697 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 10697 times.
✓ Branch 6 taken 10697 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 1287 times.
✓ Branch 9 taken 9410 times.
 21394 TASK_DELAY(ep->dtime);
6760
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9410 times.
 9410 if (xcom_shutdown) {
6761 ✗ TERMINATE;
6762 }
6763 9410 ep->dtime += CONNECT_WAIT_INCREASE; /* Increase wait time for next try */
6764
2/2
✓ Branch 0 taken 771 times.
✓ Branch 1 taken 8639 times.
 9410 if (ep->dtime > MAX_CONNECT_WAIT) {
6765 771 ep->dtime = MAX_CONNECT_WAIT;
6766 }
6767 }
6768 250828 ep->dtime = INITIAL_CONNECT_WAIT;
6769 {
6770
2/4
✓ Branch 0 taken 250828 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 250828 times.
✗ Branch 3 not taken.
 250828 unchecked_replace_pax_msg(&ep->reply, pax_msg_new_0(null_synode));
6771
6772 ADD_DBG(D_NONE, add_event(EVENT_DUMP_PAD, string_arg("ep->s->con.fd"));
6773 add_event(EVENT_DUMP_PAD, int_arg(ep->s->con.fd)););
6774
12/18
✓ Branch 0 taken 250828 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 383386 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 250817 times.
✓ Branch 5 taken 132569 times.
✓ Branch 6 taken 1694 times.
✓ Branch 7 taken 249123 times.
✓ Branch 8 taken 132569 times.
✓ Branch 9 taken 249123 times.
✗ Branch 10 not taken.
✓ Branch 11 taken 132558 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 132558 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✓ Branch 16 taken 132558 times.
✓ Branch 17 taken 249123 times.
 515944 TASK_CALL(read_msg(ep->s->con, ep->reply, ep->s, &n));
6775 ADD_DBG(D_NONE, add_event(EVENT_DUMP_PAD, string_arg("ep->s->con.fd"));
6776 add_event(EVENT_DUMP_PAD, int_arg(ep->s->con.fd)););
6777 249123 ep->reply->refcnt = 1; /* Refcnt from other end is void here */
6778
2/2
✓ Branch 0 taken 1980 times.
✓ Branch 1 taken 247143 times.
 249123 if (n <= 0) {
6779
1/2
✓ Branch 0 taken 1980 times.
✗ Branch 1 not taken.
 1980 shutdown_connection(ep->s->con);
6780 1980 continue;
6781 }
6782 247143 receive_bytes[ep->reply->op] += (uint64_t)n + MSG_HDR_SIZE;
6783 }
6784 IFDBG(D_NONE, FN; NDBG(ep->s->con.fd, d); NDBG(task_now(), f);
6785 COPY_AND_FREE_GOUT(dbg_pax_msg(ep->reply)););
6786 247143 receive_count[ep->reply->op]++;
6787
6788 ADD_DBG(D_NONE, add_synode_event(ep->reply->synode);
6789 add_event(EVENT_DUMP_PAD, string_arg("ep->reply->from"));
6790 add_event(EVENT_DUMP_PAD, uint_arg(ep->reply->from));
6791 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(ep->reply->op)));
6792 add_event(EVENT_DUMP_PAD, string_arg("get_site_def()->boot_key"));
6793 add_synode_event(get_site_def()->boot_key););
6794 /* Special test for need_snapshot, since node and site may not be
6795 * consistent
6796 */
6797
4/4
✓ Branch 0 taken 1312 times.
✓ Branch 1 taken 245831 times.
✓ Branch 2 taken 1312 times.
✓ Branch 3 taken 245831 times.
 248455 if (ep->reply->op == need_boot_op &&
6798
3/6
✓ Branch 0 taken 1312 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1312 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1312 times.
✗ Branch 5 not taken.
 1312 !synode_eq(get_site_def()->boot_key, null_synode)) {
6799 1312 pax_msg *p = ep->reply;
6800
6801 ADD_DBG(D_BASE,
6802 add_event(EVENT_DUMP_PAD,
6803 string_arg("calling server_handle_need_snapshot")););
6804
3/6
✓ Branch 0 taken 1312 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1312 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1312 times.
✗ Branch 5 not taken.
 1312 if (should_handle_need_boot(find_site_def(p->synode), p)) {
6805
2/4
✓ Branch 0 taken 1312 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1312 times.
✗ Branch 3 not taken.
 1312 server_handle_need_snapshot(ep->s, find_site_def(p->synode), p->from);
6806 /* Wake senders waiting to connect, since new node has appeared */
6807
1/2
✓ Branch 0 taken 1312 times.
✗ Branch 1 not taken.
 1312 wakeup_sender();
6808 } else {
6809 ✗ ep->s->invalid = 1;
6810 }
6811 } else {
6812 /* We only handle messages from this connection if the server is valid.
6813 */
6814
1/2
✓ Branch 0 taken 245831 times.
✗ Branch 1 not taken.
 245831 if (ep->s->invalid == 0)
6815
2/4
✓ Branch 0 taken 245831 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 245831 times.
✗ Branch 3 not taken.
 245831 dispatch_op(find_site_def(ep->reply->synode), ep->reply, nullptr);
6816 }
6817
3/6
✗ Branch 0 not taken.
✓ Branch 1 taken 247143 times.
✓ Branch 2 taken 247143 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 247143 times.
 494286 TASK_YIELD;
6818 }
6819
6820 ✗ FINALLY
6821
1/2
✓ Branch 0 taken 2981 times.
✗ Branch 1 not taken.
 2981 replace_pax_msg(&ep->reply, nullptr);
6822
6823
1/2
✓ Branch 0 taken 2981 times.
✗ Branch 1 not taken.
 2981 shutdown_connection(ep->s->con);
6824 2981 ep->s->reply_handler = nullptr;
6825 IFDBG(D_BUG, FN; STRLIT(" shutdown "); NDBG(ep->s->con.fd, d);
6826 NDBG(task_now(), f));
6827
1/2
✓ Branch 0 taken 2981 times.
✗ Branch 1 not taken.
 2981 srv_unref(ep->s);
6828
6829
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2981 times.
✓ Branch 2 taken 2981 times.
✗ Branch 3 not taken.
 2981 TASK_END;
6830 }
6831
6832 /* purecov: begin deadcode */
6833 ✗ void xcom_sleep(unsigned int seconds) {
6834 #if defined(_WIN32)
6835 Sleep((DWORD)seconds * 1000); /* windows sleep takes milliseconds */
6836 #else
6837 ✗ sleep(seconds);
6838 #endif
6839 }
6840 /* purecov: end */
6841
6842 /*
6843 * Get a unique long as the basis for XCom group id creation.
6844 *
6845 * NOTE:
6846 * As there is no gethostid() on win, we use seconds since epoch instead,
6847 * so it might fail if you try simultaneous create sites at the same second.
6848 */
6849
6850 #ifndef _WIN32
6851 #include <sys/utsname.h>
6852 #endif
6853
6854 3397 long xcom_unique_long(void) {
6855 #if defined(_WIN32)
6856 __time64_t ltime;
6857
6858 _time64(&ltime);
6859 return (long)(ltime ^ GetCurrentProcessId());
6860 #else
6861 struct utsname buf;
6862 3397 uname(&buf);
6863 3397 long id = (long)fnv_hash((unsigned char *)&buf, sizeof(buf), 0);
6864 3397 return id ^ getpid();
6865 #endif
6866 }
6867
6868 5197 app_data_ptr init_config_with_group(app_data *a, node_list *nl, cargo_type type,
6869 uint32_t group_id) {
6870 5197 init_app_data(a);
6871 5197 a->app_key.group_id = a->group_id = group_id;
6872 5197 a->body.c_t = type;
6873 5197 init_node_list(nl->node_list_len, nl->node_list_val, &a->body.app_u_u.nodes);
6874 5197 return a;
6875 }
6876
6877 9 app_data_ptr init_set_event_horizon_msg(app_data *a, uint32_t group_id,
6878 xcom_event_horizon event_horizon) {
6879 9 init_app_data(a);
6880 9 a->app_key.group_id = a->group_id = group_id;
6881 9 a->body.c_t = set_event_horizon_type;
6882 9 a->body.app_u_u.event_horizon = event_horizon;
6883 9 return a;
6884 }
6885
6886 187 app_data_ptr init_get_msg(app_data *a, uint32_t group_id, cargo_type const t) {
6887 187 init_app_data(a);
6888 187 a->app_key.group_id = a->group_id = group_id;
6889 187 a->body.c_t = t;
6890 187 return a;
6891 }
6892
6893 85 app_data_ptr init_get_leaders_msg(app_data *a, uint32_t group_id) {
6894 85 return init_get_msg(a, group_id, get_leaders_type);
6895 }
6896
6897 99 app_data_ptr init_get_event_horizon_msg(app_data *a, uint32_t group_id) {
6898 99 return init_get_msg(a, group_id, get_event_horizon_type);
6899 }
6900
6901 104115 app_data_ptr init_app_msg(app_data *a, char *payload, u_int payload_size) {
6902 104115 init_app_data(a);
6903 104115 a->body.c_t = app_type;
6904 104115 a->body.app_u_u.data.data_val = payload; /* Takes ownership of payload. */
6905 104115 a->body.app_u_u.data.data_len = payload_size;
6906 104115 return a;
6907 }
6908
6909 3 static app_data_ptr init_get_synode_app_data_msg(
6910 app_data *a, uint32_t group_id, synode_no_array *const synodes) {
6911 3 init_get_msg(a, group_id, get_synode_app_data_type);
6912 /* Move synodes (as in C++ move semantics) into a->body.app_u_u.synodes. */
6913 3 synode_array_move(&a->body.app_u_u.synodes, synodes);
6914 3 return a;
6915 }
6916
6917 3 app_data_ptr init_set_cache_size_msg(app_data *a, uint64_t cache_limit) {
6918 3 init_app_data(a);
6919 3 a->body.c_t = set_cache_limit;
6920 3 a->body.app_u_u.cache_limit = cache_limit;
6921 3 return a;
6922 }
6923
6924 ✗ app_data_ptr init_convert_into_local_server_msg(app_data *a) {
6925 ✗ init_app_data(a);
6926 ✗ a->body.c_t = convert_into_local_server_type;
6927 ✗ return a;
6928 }
6929
6930 1312 static void server_send_snapshot(server *srv, site_def const *s,
6931 gcs_snapshot *gcs_snap, node_no node) {
6932
2/4
✓ Branch 0 taken 1312 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1312 times.
✗ Branch 3 not taken.
 1312 pax_msg *p = pax_msg_new(gcs_snap->log_start, get_site_def());
6933
1/2
✓ Branch 0 taken 1312 times.
✗ Branch 1 not taken.
 1312 ref_msg(p);
6934 1312 p->op = gcs_snapshot_op;
6935 1312 p->gcs_snap = gcs_snap;
6936
2/4
✓ Branch 0 taken 1312 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1312 times.
✗ Branch 3 not taken.
 1312 send_msg(srv, s->nodeno, node, get_group_id(s), p);
6937
1/2
✓ Branch 0 taken 1312 times.
✗ Branch 1 not taken.
 1312 unref_msg(&p);
6938 1312 }
6939
6940 1312 static void server_push_log(server *srv, synode_no push, node_no node) {
6941 1312 site_def const *s = get_site_def();
6942
2/4
✓ Branch 0 taken 1312 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1312 times.
✗ Branch 3 not taken.
 1312 if (srv && s) {
6943
2/2
✓ Branch 0 taken 57735 times.
✓ Branch 1 taken 1312 times.
 59047 while (!synode_gt(push, get_max_synode())) {
6944
2/2
✓ Branch 0 taken 54672 times.
✓ Branch 1 taken 3063 times.
 57735 if (is_cached(push)) {
6945 /* Need to clone message here since pax_machine may be re-used while
6946 * message is sent */
6947 54672 pax_machine *p = get_cache_no_touch(push, FALSE);
6948
2/2
✓ Branch 0 taken 44520 times.
✓ Branch 1 taken 10152 times.
 54672 if (pm_finished(p)) {
6949
1/2
✓ Branch 0 taken 44520 times.
✗ Branch 1 not taken.
 44520 pax_msg *pm = clone_pax_msg(p->learner.msg);
6950
1/2
✓ Branch 0 taken 44520 times.
✗ Branch 1 not taken.
 44520 if (pm != nullptr) {
6951
1/2
✓ Branch 0 taken 44520 times.
✗ Branch 1 not taken.
 44520 ref_msg(pm);
6952 44520 pm->op = recover_learn_op;
6953 IFDBG(D_NONE, FN; PTREXP(srv); PTREXP(s););
6954
2/4
✓ Branch 0 taken 44520 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 44520 times.
✗ Branch 3 not taken.
 44520 send_msg(srv, s->nodeno, node, get_group_id(s), pm);
6955
1/2
✓ Branch 0 taken 44520 times.
✗ Branch 1 not taken.
 44520 unref_msg(&pm);
6956 }
6957 }
6958 }
6959 57735 push = incr_synode(push);
6960 }
6961 }
6962 1312 }
6963
6964 /* purecov: begin deadcode */
6965 ✗ static void reply_push_log(synode_no push, linkage *reply_queue) {
6966 ✗ while (!synode_gt(push, get_max_synode())) {
6967 ✗ if (is_cached(push)) {
6968 /* Need to clone message here since pax_machine may be re-used while
6969 * message is sent */
6970 ✗ pax_machine *p = get_cache_no_touch(push, FALSE);
6971 ✗ if (pm_finished(p)) {
6972 ✗ pax_msg *reply = clone_pax_msg(p->learner.msg);
6973 ✗ ref_msg(reply);
6974 ✗ reply->op = recover_learn_op;
6975 {
6976 ✗ msg_link *msg_x = msg_link_new(reply, reply->from);
6977 IFDBG(D_NONE, FN; PTREXP(msg_x));
6978 ✗ link_into(&(msg_x->l), reply_queue);
6979 }
6980 ✗ replace_pax_msg(&reply, nullptr);
6981 ✗ unref_msg(&reply);
6982 }
6983 }
6984 ✗ push = incr_synode(push);
6985 }
6986 }
6987 /* purecov: end */
6988
6989 static app_snap_getter get_app_snap_cb;
6990 static app_snap_handler handle_app_snap_cb;
6991
6992 1312 static gcs_snapshot *create_snapshot() {
6993 1312 gcs_snapshot *gs = nullptr;
6994
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1312 times.
 1312 if (get_app_snap_cb) {
6995 /* purecov: begin deadcode */
6996 ✗ blob app_snap = {{0, nullptr}}; /* Initialize in case get_app_snap_cb does
6997 not assign a value */
6998 ✗ synode_no app_lsn = get_app_snap_cb(&app_snap);
6999
7000 /* We have a valid callback, abort if it did not return anything */
7001 ✗ if (app_snap.data.data_len == 0) {
7002 ADD_DBG(D_BASE,
7003 add_event(EVENT_DUMP_PAD, string_arg("no data, return")););
7004 ✗ return nullptr;
7005 }
7006 ✗ gs = export_config();
7007 ✗ if (!gs) return nullptr;
7008 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("export config ok")););
7009 ✗ gs->app_snap = app_snap;
7010 IFDBG(D_BUG, FN; SYCEXP(app_lsn); SYCEXP(gs->log_start);
7011 SYCEXP(gs->log_end));
7012
7013 /* Set starting point of log to match the snapshot */
7014 /* If we have a valid synode from application snapshot, see if it should
7015 * be used */
7016 ✗ if (!synode_eq(null_synode, app_lsn)) {
7017 /* If log_start is null_synode, always use valid synode from application
7018 * snapshot */
7019 ✗ if (synode_eq(null_synode, gs->log_start) ||
7020 ✗ !synode_gt(app_lsn, gs->log_start)) {
7021 ✗ gs->log_start = app_lsn;
7022 IFDBG(D_BUG, FN; STRLIT("using "); SYCEXP(app_lsn));
7023 }
7024 }
7025 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("gs->log_start"));
7026 add_synode_event(gs->log_start);
7027 add_event(EVENT_DUMP_PAD, string_arg("gs->log_end"));
7028 add_synode_event(gs->log_end););
7029 /* purecov: end */
7030 } else {
7031 1312 gs = export_config();
7032
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1312 times.
 1312 if (!gs) return nullptr;
7033 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("export config ok")););
7034
1/2
✓ Branch 0 taken 1312 times.
✗ Branch 1 not taken.
 1312 if (!synode_eq(null_synode, last_config_modification_id)) {
7035 /* No valid valid synode from application snapshot, use
7036 * last_config_modification_id if not null_synode */
7037 1312 gs->log_start = last_config_modification_id;
7038 IFDBG(D_BUG, FN; STRLIT("using "); SYCEXP(last_config_modification_id));
7039 }
7040 IFDBG(D_BUG, FN; SYCEXP(gs->log_start); SYCEXP(gs->log_end));
7041 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("gs->log_start"));
7042 add_synode_event(gs->log_start);
7043 add_event(EVENT_DUMP_PAD, string_arg("gs->log_end"));
7044 add_synode_event(gs->log_end););
7045 }
7046 IFDBG(D_BUG, FN; SYCEXP(gs->log_start); SYCEXP(gs->log_end));
7047 1312 return gs;
7048 }
7049
7050 /* purecov: begin deadcode */
7051 ✗ static void handle_need_snapshot(linkage *reply_queue, pax_msg *pm) {
7052 ✗ gcs_snapshot *gs = create_snapshot();
7053 ✗ if (gs) {
7054 ✗ pax_msg *reply = clone_pax_msg(pm);
7055 ✗ ref_msg(reply);
7056 ✗ reply->op = gcs_snapshot_op;
7057 ✗ reply->gcs_snap = gs;
7058 {
7059 ✗ msg_link *msg_x = msg_link_new(reply, reply->from);
7060 IFDBG(D_NONE, FN; PTREXP(msg_x));
7061 ✗ link_into(&(msg_x->l), reply_queue);
7062 }
7063 ✗ unref_msg(&reply);
7064 IFDBG(D_NONE, FN; STRLIT("sent snapshot"););
7065 ✗ reply_push_log(gs->log_start, reply_queue);
7066 ✗ send_global_view();
7067 }
7068 }
7069 /* purecov: end */
7070
7071 static task_env *x_timer = nullptr;
7072
7073 /* Timer for use with the xcom FSM. Will deliver x_fsm_timeout */
7074 ✗ static int xcom_timer(task_arg arg) {
7075 DECL_ENV
7076 double t;
7077 ✗ ENV_INIT
7078 ✗ END_ENV_INIT
7079 END_ENV;
7080
7081 ✗ TASK_BEGIN
7082
7083 ✗ ep->t = get_double_arg(arg);
7084 ✗ TASK_DELAY(ep->t);
7085 ✗ XCOM_FSM(x_fsm_timeout, double_arg(ep->t));
7086 ✗ FINALLY
7087 ✗ if (stack == x_timer) set_task(&x_timer, nullptr);
7088 IFDBG(D_CONS, FN; STRLIT(" timeout "));
7089 ✗ TASK_END;
7090 }
7091
7092 /* Stop the xcom FSM timer */
7093 2197 static void stop_x_timer() {
7094
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2197 times.
 2197 if (x_timer) {
7095 ✗ task_terminate(x_timer);
7096 ✗ set_task(&x_timer, nullptr);
7097 }
7098 2197 }
7099
7100 /* Start the xcom FSM timer */
7101 ✗ static void start_x_timer(double t) {
7102 ✗ stop_x_timer();
7103 ✗ set_task(&x_timer, task_new(xcom_timer, double_arg(t), "xcom_timer",
7104 XCOM_THREAD_DEBUG));
7105 }
7106
7107 /* Deliver x_fsm_complete to xcom FSM */
7108 /* purecov: begin deadcode */
7109 ✗ static int x_fsm_completion_task(task_arg arg) {
7110 DECL_ENV
7111 int dummy;
7112 ✗ ENV_INIT
7113 ✗ END_ENV_INIT
7114 END_ENV;
7115
7116 ✗ TASK_BEGIN
7117
7118 (void)
7119 arg;
7120 ✗ XCOM_FSM(x_fsm_complete, null_arg);
7121 ✗ FINALLY
7122 IFDBG(D_FSM, FN; STRLIT(" delivered "));
7123 ✗ TASK_END;
7124 }
7125 /* purecov: end */
7126
7127 /* Send x_fsm_complete to xcom FSM in the context of the xcom thread. The
7128 * calling thread and the xcom thread must be in a rendezvous. Using a task to
7129 * deliver a message is an abstraction inversion, but it's the simplest
7130 * solution until we get a proper queue-based communication system going. */
7131 /* purecov: begin deadcode */
7132 ✗ void send_x_fsm_complete() {
7133 ✗ task_new(x_fsm_completion_task, null_arg, "x_fsm_completion_task",
7134 XCOM_THREAD_DEBUG);
7135 }
7136 /* purecov: end */
7137
7138 1312 static void server_handle_need_snapshot(server *srv, site_def const *s,
7139 node_no node) {
7140
2/4
✓ Branch 0 taken 1312 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1312 times.
✗ Branch 3 not taken.
 1312 G_INFO("Received an XCom snapshot request from %s:%d", srv->srv, srv->port);
7141 1312 gcs_snapshot *gs = create_snapshot();
7142
7143
1/2
✓ Branch 0 taken 1312 times.
✗ Branch 1 not taken.
 1312 if (gs) {
7144 1312 server_send_snapshot(srv, s, gs, node);
7145 IFDBG(D_NONE, FN; STRLIT("sent snapshot"););
7146
2/4
✓ Branch 0 taken 1312 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1312 times.
✗ Branch 3 not taken.
 1312 G_INFO("XCom snapshot sent to %s:%d", srv->srv, srv->port);
7147 1312 server_push_log(srv, gs->log_start, node);
7148 1312 send_global_view();
7149 }
7150 1312 }
7151
7152 #define X(b) #b
7153 const char *xcom_actions_name[] = {x_actions};
7154 #undef X
7155
7156 static int snapshots[NSERVERS];
7157
7158 /* Note that we have received snapshot from node */
7159 1360 static void note_snapshot(node_no node) {
7160
1/2
✓ Branch 0 taken 1360 times.
✗ Branch 1 not taken.
 1360 if (node != VOID_NODE_NO) {
7161 1360 snapshots[node] = 1;
7162 }
7163 1360 }
7164
7165 /* Reset set of received snapshots */
7166 3397 static void reset_snapshot_mask() {
7167 int i;
7168
2/2
✓ Branch 0 taken 339700 times.
✓ Branch 1 taken 3397 times.
 343097 for (i = 0; i < NSERVERS; i++) {
7169 339700 snapshots[i] = 0;
7170 }
7171 3397 }
7172
7173 /* See if we have got a snapshot from every node */
7174 ✗ static int got_all_snapshots() {
7175 node_no i;
7176 ✗ node_no max = get_maxnodes(get_site_def());
7177 ✗ if (0 == max) {
7178 ✗ return 0;
7179 }
7180 ✗ for (i = 0; i < max; i++) {
7181 ✗ if (!snapshots[i]) {
7182 ✗ return 0;
7183 }
7184 }
7185 ✗ return 1;
7186 }
7187
7188 static synode_no log_start_max; /* Initialized by xcom_fsm */
7189 static synode_no log_end_max; /* Initialized by xcom_fsm */
7190
7191 /* See if this snapshot is better than what we already have */
7192 /* purecov: begin deadcode */
7193 ✗ static int better_snapshot(gcs_snapshot *gcs) {
7194 ✗ synode_no boot_key = config_max_boot_key(gcs);
7195 ✗ return synode_gt(boot_key, get_site_def()->boot_key) ||
7196 ✗ (synode_eq(boot_key, get_site_def()->boot_key) &&
7197 ✗ (synode_gt(gcs->log_start, log_start_max) ||
7198 ✗ (synode_eq(gcs->log_start, log_start_max) &&
7199 ✗ synode_gt(gcs->log_end, log_end_max))));
7200 }
7201 /* purecov: end */
7202
7203 /* Install snapshot */
7204 1360 static void handle_x_snapshot(gcs_snapshot *gcs) {
7205
2/4
✓ Branch 0 taken 1360 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1360 times.
✗ Branch 3 not taken.
 1360 G_INFO(
7206 "Installing requested snapshot. Importing all incoming configurations.");
7207 1360 import_config(gcs);
7208
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1360 times.
 1360 if (get_nodeno(get_site_def()) == VOID_NODE_NO) {
7209 IFDBG(D_BASE, FN; STRLIT("Not member of site, not executing log"));
7210 ✗ gcs->log_end =
7211 gcs->log_start; /* Avoid executing log if not member of site */
7212 }
7213
1/2
✓ Branch 0 taken 1360 times.
✗ Branch 1 not taken.
 1360 if (handle_app_snap_cb)
7214 1360 handle_app_snap_cb(&gcs->app_snap, gcs->log_start, gcs->log_end);
7215 1360 set_max_synode(gcs->log_end);
7216 1360 set_executed_msg(incr_synode(gcs->log_start));
7217 1360 log_start_max = gcs->log_start;
7218 1360 log_end_max = gcs->log_end;
7219
7220 1360 set_last_received_config(get_highest_boot_key(gcs));
7221
7222
4/8
✓ Branch 0 taken 1360 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1360 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1360 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 1360 times.
✗ Branch 7 not taken.
 1360 G_INFO("Finished snapshot installation. My node number is %d",
7223 get_nodeno(get_site_def()));
7224
7225 IFDBG(D_BUG, FN; SYCEXP(gcs->log_start); SYCEXP(gcs->log_end);
7226 SYCEXP(last_config_modification_id); SYCEXP(executed_msg););
7227 1360 }
7228
7229 /* Note that we have received snapshot, and install if better than old */
7230 /* purecov: begin deadcode */
7231 ✗ static void update_best_snapshot(gcs_snapshot *gcs) {
7232 ✗ if (get_site_def() == nullptr || better_snapshot(gcs)) {
7233 ✗ handle_x_snapshot(gcs);
7234 }
7235 }
7236 /* purecov: end */
7237
7238 /* Send need_boot_op to all nodes in current config */
7239 /* purecov: begin deadcode */
7240 ✗ static void send_need_boot() {
7241 ✗ pax_msg *p = pax_msg_new_0(null_synode);
7242 ✗ ref_msg(p);
7243 ✗ p->synode = get_site_def()->start;
7244 ✗ p->op = need_boot_op;
7245 ✗ send_to_all_except_self(get_site_def(), p, "need_boot_op");
7246 ✗ unref_msg(&p);
7247 }
7248 /* purecov: end */
7249
7250 /* Set log_end of snapshot based on log_end in snapshot and max synode */
7251 2672 void set_log_end(gcs_snapshot *gcs) {
7252
2/2
✓ Branch 0 taken 1312 times.
✓ Branch 1 taken 1360 times.
 2672 if (synode_gt(get_max_synode(), gcs->log_end)) {
7253 1312 gcs->log_end = get_max_synode();
7254 }
7255 2672 }
7256
7257 struct xcom_fsm_state;
7258 typedef struct xcom_fsm_state xcom_fsm_state;
7259
7260 /* Function pointer corresponding to a state. Return 1 if execution should
7261 * continue, 0 otherwise */
7262 typedef int (*xcom_fsm_fp)(xcom_actions action, task_arg fsmargs,
7263 xcom_fsm_state *ctxt);
7264
7265 /* Function pointer and name */
7266 struct xcom_fsm_state {
7267 xcom_fsm_fp state_fp;
7268 char const *state_name;
7269 };
7270
7271 #define X_FSM_STATE(s) \
7272 { s, #s }
7273 #define SET_X_FSM_STATE(s) \
7274 do { \
7275 ctxt->state_fp = s; \
7276 ctxt->state_name = #s; \
7277 } while (0)
7278
7279 /* The state functions/thunks */
7280 static int xcom_fsm_init(xcom_actions action, task_arg fsmargs,
7281 xcom_fsm_state *ctxt);
7282 static int xcom_fsm_start_enter(xcom_actions action, task_arg fsmargs,
7283 xcom_fsm_state *ctxt);
7284 static int xcom_fsm_start(xcom_actions action, task_arg fsmargs,
7285 xcom_fsm_state *ctxt);
7286 static int xcom_fsm_snapshot_wait_enter(xcom_actions action, task_arg fsmargs,
7287 xcom_fsm_state *ctxt);
7288 static int xcom_fsm_snapshot_wait(xcom_actions action, task_arg fsmargs,
7289 xcom_fsm_state *ctxt);
7290 static int xcom_fsm_recover_wait_enter(xcom_actions action, task_arg fsmargs,
7291 xcom_fsm_state *ctxt);
7292 static int xcom_fsm_recover_wait(xcom_actions action, task_arg fsmargs,
7293 xcom_fsm_state *ctxt);
7294 static int xcom_fsm_run_enter(xcom_actions action, task_arg fsmargs,
7295 xcom_fsm_state *ctxt);
7296 static int xcom_fsm_run(xcom_actions action, task_arg fsmargs,
7297 xcom_fsm_state *ctxt);
7298
7299 /* You are in a twisting maze of little functions ... */
7300
7301 /* init state */
7302 1207 static int xcom_fsm_init(xcom_actions action, task_arg fsmargs,
7303 xcom_fsm_state *ctxt) {
7304 (void)action;
7305 (void)fsmargs;
7306 IFDBG(D_NONE, FN;);
7307 /* Initialize basic xcom data */
7308 1207 xcom_thread_init();
7309 1207 SET_X_FSM_STATE(xcom_fsm_start_enter);
7310 1207 return 1;
7311 }
7312
7313 /* start_enter state */
7314 3397 static int xcom_fsm_start_enter(xcom_actions action, task_arg fsmargs,
7315 xcom_fsm_state *ctxt) {
7316 (void)action;
7317 (void)fsmargs;
7318 /* push_dbg(D_DETECT | D_FSM | D_FILEOP | D_CONS | D_BASE | D_TRANSPORT);
7319 */
7320 3397 push_dbg(D_FSM);
7321 IFDBG(D_NONE, FN; STRLIT("state x_start"););
7322 3397 empty_prop_input_queue();
7323 3397 empty_synode_number_pool();
7324 3397 reset_snapshot_mask();
7325 3397 set_last_received_config(null_synode);
7326
7327 3397 SET_X_FSM_STATE(xcom_fsm_start);
7328 3397 return 1;
7329 }
7330
7331 837 static int handle_fsm_net_boot(task_arg fsmargs, xcom_fsm_state *ctxt,
7332 int cont) {
7333 837 app_data *a = (app_data *)get_void_arg(fsmargs);
7334 837 install_node_group(a);
7335
1/2
✓ Branch 0 taken 837 times.
✗ Branch 1 not taken.
 837 if (is_member(get_site_def())) {
7336 837 empty_prop_input_queue();
7337 837 empty_synode_number_pool();
7338 {
7339
1/2
✓ Branch 0 taken 837 times.
✗ Branch 1 not taken.
 837 synode_no start = get_site_def()->start;
7340
1/2
✓ Branch 0 taken 837 times.
✗ Branch 1 not taken.
 837 if (start.msgno == 0) { /* May happen during initial boot */
7341 837 start.msgno = 1; /* Start with first xcom message */
7342 /* If msgno is 0, it means that this node installed a unified_boot
7343 which came from the client, thus this node is the one that will send
7344 the unified_boot on xcom, so set the node number of start accordingly
7345 */
7346
2/4
✓ Branch 0 taken 837 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 837 times.
✗ Branch 3 not taken.
 837 start.node = get_nodeno(get_site_def());
7347 }
7348
1/2
✓ Branch 0 taken 837 times.
✗ Branch 1 not taken.
 837 set_executed_msg(start);
7349 }
7350 837 pop_dbg();
7351 837 SET_X_FSM_STATE(xcom_fsm_run_enter);
7352 837 cont = 1;
7353 }
7354 837 return cont;
7355 }
7356
7357 1360 static int handle_fsm_snapshot(task_arg fsmargs, xcom_fsm_state *ctxt) {
7358 1360 gcs_snapshot *gcs = (gcs_snapshot *)get_void_arg(fsmargs);
7359 1360 empty_prop_input_queue();
7360 1360 empty_synode_number_pool();
7361 1360 set_log_end(gcs);
7362 1360 handle_x_snapshot(gcs);
7363
7364 /* Get recovery manager going again */
7365
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1360 times.
 1360 if (recovery_restart_cb) recovery_restart_cb();
7366
7367 /* If we run under control of the recovery manager, we need to call
7368 * recovery_begin_cb to rendezvous with the recovery manager */
7369
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1360 times.
 1360 if (recovery_begin_cb) recovery_begin_cb();
7370
7371 /* If we run under control of the recovery manager, we need to call
7372 * recovery_end_cb to rendezvous with the recovery manager */
7373
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1360 times.
 1360 if (recovery_end_cb) recovery_end_cb();
7374
7375 /* If we are here, it means that we are recovering from another node
7376 */
7377 /* Do not bother to wait for more snapshots, just handle it and
7378 enter run state */
7379 1360 pop_dbg();
7380 1360 SET_X_FSM_STATE(xcom_fsm_run_enter);
7381 1360 return 1;
7382 }
7383
7384 /* purecov: begin deadcode */
7385 ✗ static int handle_fsm_snapshot_wait(xcom_fsm_state *ctxt) {
7386 ✗ empty_prop_input_queue();
7387 ✗ empty_synode_number_pool();
7388 ✗ start_x_timer(SNAPSHOT_WAIT_TIME);
7389 ✗ pop_dbg();
7390 ✗ SET_X_FSM_STATE(xcom_fsm_snapshot_wait_enter);
7391 ✗ return 1;
7392 }
7393 /* purecov: end */
7394
7395 2348 static void handle_fsm_exit() {
7396 /* Xcom is finished when we get here */
7397 2348 push_dbg(D_BUG);
7398 2348 bury_site(get_group_id(get_site_def()));
7399 2348 task_terminate_all(); /* Kill, kill, kill, kill, kill, kill. This is
7400 the end. */
7401
7402 /* init_xcom_base(); */ /* Reset shared variables */
7403 2348 init_tasks(); /* Reset task variables */
7404 2348 free_site_defs();
7405 2348 free_forced_config_site_def();
7406 2348 wait_forced_config = 0;
7407 2348 garbage_collect_servers();
7408 IFDBG(D_NONE, FN; STRLIT("shutting down"));
7409 2348 xcom_shutdown = 1;
7410 2348 start_config = null_synode;
7411
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 2347 times.
 2348 G_DEBUG("Exiting xcom thread");
7412 2348 }
7413
7414 /* start state */
7415 9255 static int xcom_fsm_start(xcom_actions action, task_arg fsmargs,
7416 xcom_fsm_state *ctxt) {
7417 static int need_init_cache = 0;
7418 9255 int cont = 0; /* Set to 1 if we should continue execution */
7419
7420
5/6
✓ Branch 0 taken 2362 times.
✓ Branch 1 taken 837 times.
✓ Branch 2 taken 1360 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2348 times.
✓ Branch 5 taken 2348 times.
 9255 switch (action) {
7421 2362 case x_fsm_init:
7422 2362 xcom_shutdown = 0;
7423 2362 sent_alive = 0.0;
7424 2362 oom_abort = 0;
7425
2/2
✓ Branch 0 taken 1155 times.
✓ Branch 1 taken 1207 times.
 2362 if (need_init_cache) init_cache();
7426 2362 break;
7427
7428 837 case x_fsm_net_boot:
7429 837 cont = handle_fsm_net_boot(fsmargs, ctxt, cont);
7430 837 break;
7431
7432 1360 case x_fsm_snapshot:
7433 1360 cont = handle_fsm_snapshot(fsmargs, ctxt);
7434 1360 break;
7435
7436 /* This is the entry point for the initial recovery after the process
7437 * has started when running under an external recovery manager. */
7438 /* If we get x_fsm_snapshot_wait, we are called from the recovery
7439 * manager thread */
7440 /* purecov: begin deadcode */
7441 ✗ case x_fsm_snapshot_wait:
7442 ✗ cont = handle_fsm_snapshot_wait(ctxt);
7443 ✗ break;
7444 /* purecov: end */
7445
7446 2348 case x_fsm_exit:
7447 2348 handle_fsm_exit();
7448 2348 break;
7449
7450 2348 default:
7451 2348 break;
7452 }
7453 9255 need_init_cache = 1;
7454 9255 return cont;
7455 }
7456
7457 /* snapshot_wait_enter state */
7458 /* purecov: begin deadcode */
7459 ✗ static int xcom_fsm_snapshot_wait_enter(xcom_actions action, task_arg fsmargs,
7460 xcom_fsm_state *ctxt) {
7461 (void)action;
7462 (void)fsmargs;
7463 ✗ push_dbg(D_DETECT | D_FSM | D_FILEOP | D_CONS | D_BASE | D_TRANSPORT);
7464 IFDBG(D_NONE, FN; STRLIT("state x_snapshot_wait"););
7465 ✗ log_start_max = null_synode;
7466 ✗ log_end_max = null_synode;
7467 ✗ SET_X_FSM_STATE(xcom_fsm_snapshot_wait);
7468 ✗ return 0;
7469 }
7470 /* purecov: end */
7471
7472 /* purecov: begin deadcode */
7473 ✗ static int handle_local_snapshot(task_arg fsmargs, xcom_fsm_state *ctxt) {
7474 ✗ update_best_snapshot((gcs_snapshot *)get_void_arg(fsmargs));
7475 /* When recovering locally, fetch node number from site_def after
7476 * processing the snapshot */
7477 ✗ note_snapshot(get_site_def()->nodeno);
7478 ✗ send_need_boot();
7479 ✗ pop_dbg();
7480 ✗ SET_X_FSM_STATE(xcom_fsm_recover_wait_enter);
7481 ✗ return 1;
7482 }
7483 /* purecov: end */
7484
7485 /* purecov: begin deadcode */
7486 ✗ static int handle_snapshot(task_arg fsmargs, xcom_fsm_state *ctxt) {
7487 /* Snapshot from another node */
7488 ✗ gcs_snapshot *gcs = (gcs_snapshot *)get_void_arg(fsmargs);
7489 ✗ set_log_end(gcs);
7490 ✗ update_best_snapshot(gcs);
7491 /* We now have a site, so note that we have processed the local
7492 * snapshot even if we have not seen one, since if we are here, no
7493 * local snapshot will ever arrive. This simplifies the test in
7494 * got_all_snapshots() */
7495 ✗ note_snapshot(get_site_def()->nodeno);
7496 ✗ send_need_boot();
7497 ✗ pop_dbg();
7498 ✗ SET_X_FSM_STATE(xcom_fsm_recover_wait_enter);
7499 ✗ return 1;
7500 }
7501 /* purecov: end */
7502
7503 /* snapshot_wait state */
7504 /* purecov: begin deadcode */
7505 ✗ static int xcom_fsm_snapshot_wait(xcom_actions action, task_arg fsmargs,
7506 xcom_fsm_state *ctxt) {
7507 ✗ switch (action) {
7508 /* If we get x_fsm_local_snapshot, we are called from the recovery
7509 * manager thread */
7510 ✗ case x_fsm_local_snapshot:
7511 ✗ return handle_local_snapshot(fsmargs, ctxt);
7512
7513 ✗ case x_fsm_snapshot:
7514 ✗ return handle_snapshot(fsmargs, ctxt);
7515
7516 ✗ case x_fsm_timeout:
7517 /* Will time out if no snapshot available */
7518 /* If we run under control of the recovery manager, we need to call
7519 * recovery_end_cb to rendezvous with the recovery manager */
7520 ✗ if (recovery_end_cb) recovery_end_cb();
7521 ✗ pop_dbg();
7522 ✗ SET_X_FSM_STATE(xcom_fsm_start_enter);
7523 ✗ return 1;
7524
7525 ✗ default:
7526 ✗ break;
7527 }
7528 ✗ return 0;
7529 }
7530 /* purecov: end */
7531
7532 /* recover_wait_enter state */
7533 /* purecov: begin deadcode */
7534 ✗ static int xcom_fsm_recover_wait_enter(xcom_actions action, task_arg fsmargs,
7535 xcom_fsm_state *ctxt) {
7536 (void)action;
7537 (void)fsmargs;
7538 ✗ push_dbg(D_DETECT | D_FSM | D_FILEOP | D_CONS | D_BASE | D_TRANSPORT);
7539 IFDBG(D_NONE, FN; STRLIT("state x_recover_wait"););
7540 ✗ if (got_all_snapshots()) {
7541 /* Need to send message to trigger transition in context of xcom
7542 * thread */
7543 ✗ send_x_fsm_complete();
7544 }
7545 ✗ SET_X_FSM_STATE(xcom_fsm_recover_wait);
7546 ✗ return 0;
7547 }
7548 /* purecov: end */
7549
7550 /* recover_wait state */
7551 /* purecov: begin deadcode */
7552 ✗ static int xcom_fsm_recover_wait(xcom_actions action, task_arg fsmargs,
7553 xcom_fsm_state *ctxt) {
7554 ✗ if (action == x_fsm_snapshot) {
7555 ✗ gcs_snapshot *gcs = (gcs_snapshot *)get_void_arg(fsmargs);
7556 ✗ set_log_end(gcs);
7557 ✗ update_best_snapshot(gcs);
7558 ✗ } else if (action == x_fsm_timeout || action == x_fsm_complete) {
7559 /* Wait terminated by timeout or because all nodes have sent a
7560 * snapshot */
7561 /* If we run under control of the recovery manager, we need to call
7562 * recovery_end_cb to rendezvous with the recovery manager */
7563 ✗ if (recovery_end_cb) recovery_end_cb();
7564 ✗ pop_dbg();
7565 ✗ SET_X_FSM_STATE(xcom_fsm_run_enter);
7566 ✗ return 1;
7567 }
7568 ✗ if (got_all_snapshots()) {
7569 /* Need to send message to trigger transition in context of xcom
7570 * thread */
7571 ✗ send_x_fsm_complete();
7572 }
7573 ✗ return 0;
7574 }
7575 /* purecov: end */
7576
7577 /* run_enter state */
7578 2197 static int xcom_fsm_run_enter(xcom_actions action, task_arg fsmargs,
7579 xcom_fsm_state *ctxt) {
7580 (void)action;
7581 (void)fsmargs;
7582 2197 start_config = get_site_def()->boot_key;
7583
7584 /* Final sanity check of executed_msg */
7585
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2197 times.
 2197 if (find_site_def(executed_msg) == nullptr) {
7586 /* No site_def matches executed_msg, set it to site->start */
7587 ✗ set_executed_msg(get_site_def()->start);
7588 }
7589
7590 IFDBG(D_NONE, FN; STRLIT("state x_run"););
7591 IFDBG(D_BUG, FN; SYCEXP(executed_msg););
7592 IFDBG(D_BUG, FN; SYCEXP(start_config););
7593 2197 stop_x_timer();
7594
1/2
✓ Branch 0 taken 2197 times.
✗ Branch 1 not taken.
 2197 if (xcom_run_cb) xcom_run_cb(0);
7595 2197 client_boot_done = 1;
7596 2197 netboot_ok = 1;
7597 2197 set_proposer_startpoint();
7598 2197 create_proposers();
7599 2197 set_task(&executor, task_new(executor_task, null_arg, "executor_task",
7600 XCOM_THREAD_DEBUG));
7601 2197 set_task(&sweeper,
7602 task_new(sweeper_task, null_arg, "sweeper_task", XCOM_THREAD_DEBUG));
7603 2197 set_task(&detector, task_new(detector_task, null_arg, "detector_task",
7604 XCOM_THREAD_DEBUG));
7605 2197 set_task(&alive_t,
7606 task_new(alive_task, null_arg, "alive_task", XCOM_THREAD_DEBUG));
7607 2197 set_task(&cache_task, task_new(cache_manager_task, null_arg,
7608 "cache_manager_task", XCOM_THREAD_DEBUG));
7609
7610 2197 push_dbg(D_FSM /* | D_EXEC | D_BASE | D_TRANSPORT */);
7611 2197 SET_X_FSM_STATE(xcom_fsm_run);
7612 2197 return 1;
7613 }
7614
7615 2190 static int handle_fsm_terminate(task_arg fsmargs, xcom_fsm_state *ctxt) {
7616 2190 dump_debug_exec_state();
7617 2190 client_boot_done = 0;
7618 2190 netboot_ok = 0;
7619 2190 oom_abort = 0;
7620 2190 terminate_proposers();
7621 2190 init_proposers();
7622 2190 task_terminate(executor);
7623 2190 set_task(&executor, nullptr);
7624 2190 task_terminate(sweeper);
7625 2190 set_task(&sweeper, nullptr);
7626 2190 task_terminate(detector);
7627 2190 set_task(&detector, nullptr);
7628 2190 task_terminate(alive_t);
7629 2190 set_task(&alive_t, nullptr);
7630 2190 task_terminate(cache_task);
7631 2190 set_task(&cache_task, nullptr);
7632
7633 2190 init_xcom_base(); /* Reset shared variables */
7634 2190 free_site_defs();
7635 2190 free_forced_config_site_def();
7636 2190 wait_forced_config = 0;
7637 2190 garbage_collect_servers();
7638
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2190 times.
 2190 if (xcom_terminate_cb) xcom_terminate_cb(get_int_arg(fsmargs));
7639 2190 pop_dbg();
7640 2190 SET_X_FSM_STATE(xcom_fsm_start_enter);
7641 2190 return 1;
7642 }
7643
7644 27 static void handle_fsm_force_config(task_arg fsmargs) {
7645 27 app_data *a = (app_data *)get_void_arg(fsmargs);
7646 27 site_def *s = create_site_def_with_start(a, executed_msg);
7647
7648 27 s->boot_key = executed_msg;
7649 27 invalidate_servers(get_site_def(), s);
7650 27 start_force_config(s, 1);
7651 27 wait_forced_config = 1; /* Note that forced config has not yet arrived */
7652 27 }
7653
7654 /* run state */
7655 5251 static int xcom_fsm_run(xcom_actions action, task_arg fsmargs,
7656 xcom_fsm_state *ctxt) {
7657
3/4
✓ Branch 0 taken 2190 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 27 times.
✓ Branch 3 taken 3034 times.
 5251 switch (action) {
7658 2190 case x_fsm_terminate:
7659 2190 return handle_fsm_terminate(fsmargs, ctxt);
7660
7661 /* purecov: begin deadcode */
7662 ✗ case x_fsm_need_snapshot:
7663 IFDBG(D_NONE, STRLIT("got snapshot request in x_run state"));
7664 ✗ break;
7665 /* purecov: end */
7666
7667 27 case x_fsm_force_config:
7668 27 handle_fsm_force_config(fsmargs);
7669 27 break;
7670
7671 3034 default:
7672 3034 break;
7673 }
7674 3061 return 0;
7675 }
7676
7677 /* Trampoline which loops calling thunks pointed to by ctxt.state_fp until 0
7678 * is returned. Return pointer to ctxt. */
7679 10119 xcom_fsm_state *xcom_fsm_impl(xcom_actions action, task_arg fsmargs) {
7680 static xcom_fsm_state ctxt = X_FSM_STATE(xcom_fsm_init);
7681
7682
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 10112 times.
 10119 G_DEBUG("%f pid %d xcom_id %x state %s action %s", seconds(), xpid(),
7683 get_my_xcom_id(), ctxt.state_name, xcom_actions_name[action]);
7684 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("state"));
7685 add_event(EVENT_DUMP_PAD, string_arg(ctxt.state_name));
7686 add_event(EVENT_DUMP_PAD, string_arg("action"));
7687 add_event(EVENT_DUMP_PAD, string_arg(xcom_actions_name[action]));
7688 add_event(EVENT_DUMP_PAD, string_arg("executed_msg"));
7689 add_synode_event(executed_msg););
7690 #ifdef TASK_EVENT_TRACE
7691 dump_task_events();
7692 #endif
7693 /* Crank the state machine until it stops */
7694 IFDBG(D_BUG, FN; STREXP(ctxt.state_name); STREXP(xcom_actions_name[action]));
7695
2/2
✓ Branch 0 taken 11188 times.
✓ Branch 1 taken 10119 times.
 21307 while (ctxt.state_fp(action, fsmargs, &ctxt)) {
7696 IFDBG(D_BUG, FN; STREXP(ctxt.state_name);
7697 STREXP(xcom_actions_name[action]));
7698 }
7699 10119 return &ctxt;
7700 }
7701
7702 /* Call FSM trampoline and return state name of resulting state */
7703 10119 char const *xcom_fsm(xcom_actions action, task_arg fsmargs) {
7704 10119 xcom_fsm_state *s = xcom_fsm_impl(action, fsmargs);
7705 10119 return s->state_name;
7706 }
7707
7708 /* See if we can send a snapshot to another node */
7709 /* purecov: begin deadcode */
7710 ✗ static int can_send_snapshot() {
7711 ✗ xcom_fsm_state *state = xcom_fsm_impl(x_fsm_need_snapshot, null_arg);
7712 ✗ return state->state_fp == xcom_fsm_run;
7713 }
7714 /* purecov: end */
7715
7716 2302 void set_app_snap_handler(app_snap_handler x) { handle_app_snap_cb = x; }
7717
7718 /* purecov: begin deadcode */
7719 ✗ void set_app_snap_getter(app_snap_getter x) { get_app_snap_cb = x; }
7720 /* purecov: end */
7721
7722 /* Read max n bytes from socket fd into buffer buf */
7723 5717 static result socket_read(connection_descriptor *rfd, void *buf, int n) {
7724 5717 result ret = {0, 0};
7725
7726
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5717 times.
 5717 assert(n >= 0);
7727
7728 do {
7729
1/2
✓ Branch 0 taken 20703 times.
✗ Branch 1 not taken.
 20703 ret = con_read(rfd, buf, n);
7730 20703 task_dump_err(ret.funerr);
7731
5/6
✓ Branch 0 taken 14986 times.
✓ Branch 1 taken 5717 times.
✓ Branch 2 taken 14986 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 14986 times.
✓ Branch 5 taken 5717 times.
 20703 } while (ret.val < 0 && can_retry_read(ret.funerr));
7732 5717 return ret;
7733 }
7734
7735 /* Read exactly n bytes from socket fd into buffer buf */
7736 5717 static int64_t socket_read_bytes(connection_descriptor *rfd, char *p,
7737 uint32_t n) {
7738 5717 uint32_t left = n;
7739 5717 char *bytes = p;
7740
7741 5717 result nread = {0, 0};
7742
7743
2/2
✓ Branch 0 taken 5717 times.
✓ Branch 1 taken 5517 times.
 11234 while (left > 0) {
7744 /*
7745 socket_read just reads no more than INT_MAX bytes. We should not pass
7746 a length more than INT_MAX to it.
7747 */
7748 5717 int r = (int)MIN(left, INT_MAX);
7749
7750
1/2
✓ Branch 0 taken 5717 times.
✗ Branch 1 not taken.
 5717 nread = socket_read(rfd, bytes, r);
7751
2/2
✓ Branch 0 taken 200 times.
✓ Branch 1 taken 5517 times.
 5717 if (nread.val == 0) {
7752 200 return 0;
7753
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5517 times.
 5517 } else if (nread.val < 0) {
7754 ✗ return -1;
7755 } else {
7756 5517 bytes += nread.val;
7757 5517 left -= (uint32_t)nread.val;
7758 }
7759 }
7760
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5517 times.
 5517 assert(left == 0);
7761 5517 return n;
7762 }
7763
7764 /* Write n bytes from buffer buf to socket fd */
7765 111339 static int64_t socket_write(connection_descriptor *wfd, void *_buf, uint32_t n,
7766 connnection_write_method write_function) {
7767 111339 char *buf = (char *)_buf;
7768 111339 result ret = {0, 0};
7769
7770 uint32_t total; /* Keeps track of number of bytes written so far */
7771
7772 111339 total = 0;
7773
2/2
✓ Branch 0 taken 111340 times.
✓ Branch 1 taken 110236 times.
 221576 while (total < n) {
7774 111340 int w = (int)MIN(n - total, INT_MAX);
7775
7776
5/8
✓ Branch 0 taken 111339 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1102 times.
✓ Branch 3 taken 110237 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 1102 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 111339 times.
 112442 while ((ret = write_function(wfd, buf + total, w)).val < 0 &&
7777 1102 can_retry_write(ret.funerr)) {
7778 ✗ task_dump_err(ret.funerr);
7779 IFDBG(D_NONE, FN; STRLIT("retry "); NEXP(total, d); NEXP(n, d));
7780 }
7781
2/2
✓ Branch 0 taken 1102 times.
✓ Branch 1 taken 110237 times.
 111339 if (ret.val <= 0) { /* Something went wrong */
7782 1102 task_dump_err(ret.funerr);
7783 1102 return -1;
7784 } else {
7785 110237 total += (uint32_t)ret.val; /* Add number of bytes written to total */
7786 }
7787 }
7788 IFDBG(D_TRANSPORT, FN; NEXP(total, u); NEXP(n, u));
7789
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 110236 times.
 110236 assert(total == n);
7790 110236 return (total);
7791 }
7792
7793 #define CONNECT_FAIL \
7794 ret_fd = -1; \
7795 goto end
7796
7797 ✗ connection_descriptor *xcom_open_client_connection(char const *server,
7798 xcom_port port) {
7799 ✗ return open_new_connection(server, port);
7800 }
7801
7802 /* Send a protocol negotiation message on connection con */
7803 2039 static int xcom_send_proto(connection_descriptor *con, xcom_proto x_proto,
7804 x_msg_type x_type, unsigned int tag) {
7805 char buf[MSG_HDR_SIZE];
7806 2039 memset(buf, 0, MSG_HDR_SIZE);
7807
7808
1/2
✓ Branch 0 taken 2039 times.
✗ Branch 1 not taken.
 2039 if (con->fd >= 0) {
7809 2039 con->snd_tag = tag;
7810
1/2
✓ Branch 0 taken 2039 times.
✗ Branch 1 not taken.
 2039 write_protoversion(VERS_PTR((unsigned char *)buf), x_proto);
7811
1/2
✓ Branch 0 taken 2039 times.
✗ Branch 1 not taken.
 2039 put_header_1_0((unsigned char *)buf, 0, x_type, tag);
7812 {
7813 int sent;
7814
1/2
✓ Branch 0 taken 2039 times.
✗ Branch 1 not taken.
 2039 sent = (int)socket_write(con, buf, MSG_HDR_SIZE);
7815
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2039 times.
 2039 if (con->fd < 0) {
7816 ✗ return -1;
7817 }
7818 2039 return sent;
7819 }
7820 } else {
7821 ✗ return -1;
7822 }
7823 }
7824
7825 2039 static int xcom_recv_proto(connection_descriptor *rfd, xcom_proto *x_proto,
7826 x_msg_type *x_type, unsigned int *tag) {
7827 int n;
7828 unsigned char header_buf[MSG_HDR_SIZE];
7829 uint32_t msgsize;
7830
7831 /* Read length field, protocol version, and checksum */
7832
1/2
✓ Branch 0 taken 2039 times.
✗ Branch 1 not taken.
 2039 n = (int)socket_read_bytes(rfd, (char *)header_buf, MSG_HDR_SIZE);
7833
7834
2/2
✓ Branch 0 taken 200 times.
✓ Branch 1 taken 1839 times.
 2039 if (n != MSG_HDR_SIZE) {
7835 IFDBG(D_NONE, FN; NDBG(n, d));
7836 200 return -1;
7837 }
7838
7839
1/2
✓ Branch 0 taken 1839 times.
✗ Branch 1 not taken.
 1839 *x_proto = read_protoversion(VERS_PTR(header_buf));
7840
1/2
✓ Branch 0 taken 1839 times.
✗ Branch 1 not taken.
 1839 get_header_1_0(header_buf, &msgsize, x_type, tag);
7841
7842 1839 return n;
7843 }
7844
7845 enum { TAG_START = 313 };
7846
7847 /**
7848 * @brief Checks if a given app_data is from a given cargo_type.
7849 *
7850 * @param a the app_data
7851 * @param t the cargo type
7852 * @return int TRUE (1) if app_data a is from cargo_type t
7853 */
7854
7855 1851 static inline int is_cargo_type(app_data_ptr a, cargo_type t) {
7856
3/4
✓ Branch 0 taken 1851 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1824 times.
✓ Branch 3 taken 27 times.
 1851 return a ? (a->body.c_t == t) : 0;
7857 }
7858
7859 /**
7860 * @brief Retrieves the address that was used in the add_node request
7861 *
7862 * @param a app data containing the node to add
7863 * @param member address we used to present ourselves to other nodes
7864 * @return char* a pointer to the address being added.
7865 */
7866 12 static char *get_add_node_address(app_data_ptr a, unsigned int *member) {
7867 12 char *retval = nullptr;
7868
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 12 times.
 12 if (!is_cargo_type(a, add_node_type)) return nullptr;
7869
7870
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 3 times.
 12 if ((*member) < a->body.app_u_u.nodes.node_list_len) {
7871 9 retval = a->body.app_u_u.nodes.node_list_val[(*member)].address;
7872 9 (*member)++;
7873 }
7874
7875 12 return retval;
7876 }
7877
7878 10756 int is_node_v4_reachable_with_info(struct addrinfo *retrieved_addr_info) {
7879 10756 int v4_reachable = 0;
7880
7881 /* Verify if we are reachable either by V4 and by V6 with the provided
7882 address. */
7883 10756 struct addrinfo *my_own_information_loop = nullptr;
7884
7885 10756 my_own_information_loop = retrieved_addr_info;
7886
4/4
✓ Branch 0 taken 10763 times.
✓ Branch 1 taken 10749 times.
✓ Branch 2 taken 10756 times.
✓ Branch 3 taken 7 times.
 21512 while (!v4_reachable && my_own_information_loop) {
7887
2/2
✓ Branch 0 taken 10749 times.
✓ Branch 1 taken 7 times.
 10756 if (my_own_information_loop->ai_family == AF_INET) {
7888 10749 v4_reachable = 1;
7889 }
7890 10756 my_own_information_loop = my_own_information_loop->ai_next;
7891 }
7892
7893 10756 return v4_reachable;
7894 }
7895
7896 6 int is_node_v4_reachable(char *node_address) {
7897 6 int v4_reachable = 0;
7898
7899 /* Verify if we are reachable either by V4 and by V6 with the provided
7900 address. */
7901 6 struct addrinfo *my_own_information = nullptr;
7902
7903
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 checked_getaddrinfo(node_address, nullptr, nullptr, &my_own_information);
7904
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6 times.
 6 if (my_own_information == nullptr) {
7905 ✗ return v4_reachable;
7906 }
7907
7908
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 v4_reachable = is_node_v4_reachable_with_info(my_own_information);
7909
7910
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 if (my_own_information) freeaddrinfo(my_own_information);
7911
7912 6 return v4_reachable;
7913 }
7914
7915 9 int are_we_allowed_to_upgrade_to_v6(app_data_ptr a) {
7916 /* This should the address we used to present ourselves to other nodes. */
7917 9 unsigned int list_member = 0;
7918 9 char *added_node = nullptr;
7919
7920 9 int is_v4_reachable = 0;
7921
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 3 times.
 12 while ((added_node = get_add_node_address(a, &list_member)) != nullptr) {
7922 xcom_port my_own_port;
7923 char my_own_address[IP_MAX_SIZE];
7924 int ip_and_port_error =
7925
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 get_ip_and_port(added_node, my_own_address, &my_own_port);
7926
7927
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 6 times.
 9 if (ip_and_port_error) {
7928
3/6
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 3 times.
✗ Branch 5 not taken.
 3 G_DEBUG("Error retrieving IP and Port information");
7929 6 return 0;
7930 }
7931
7932 /* Verify if we are reachable either by V4 and by V6 with the provided
7933 address.
7934 This means that the other side won't be able to contact us since we
7935 do not provide a public V4 address */
7936
3/4
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
✓ Branch 3 taken 3 times.
 6 if (!(is_v4_reachable = is_node_v4_reachable(my_own_address))) {
7937
2/4
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
 3 G_ERROR(
7938 "Unable to add node to a group of older nodes. Please "
7939 "reconfigure "
7940 "you local address to an IPv4 address or configure your DNS to "
7941 "provide "
7942 "an IPv4 address");
7943 3 return 0;
7944 }
7945 }
7946
7947 3 return is_v4_reachable;
7948 }
7949
7950 2042 int64_t xcom_send_client_app_data(connection_descriptor *fd, app_data_ptr a,
7951 int force) {
7952
1/2
✓ Branch 0 taken 2042 times.
✗ Branch 1 not taken.
 2042 pax_msg *msg = pax_msg_new(null_synode, nullptr);
7953 2042 uint32_t buflen = 0;
7954 2042 char *buf = nullptr;
7955 2042 int64_t retval = 0;
7956 2042 int serialized = 0;
7957
7958
2/2
✓ Branch 0 taken 2039 times.
✓ Branch 1 taken 3 times.
 2042 if (!proto_done(fd)) {
7959 xcom_proto x_proto;
7960 x_msg_type x_type;
7961 unsigned int tag;
7962
1/2
✓ Branch 0 taken 2039 times.
✗ Branch 1 not taken.
 2039 retval = xcom_send_proto(fd, my_xcom_version, x_version_req, TAG_START);
7963
4/6
✓ Branch 0 taken 2039 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 2038 times.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
 2039 G_DEBUG("client sent negotiation request for protocol %d", my_xcom_version);
7964
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2039 times.
 2239 if (retval < 0) goto end;
7965
1/2
✓ Branch 0 taken 2039 times.
✗ Branch 1 not taken.
 2039 retval = xcom_recv_proto(fd, &x_proto, &x_type, &tag);
7966
2/2
✓ Branch 0 taken 200 times.
✓ Branch 1 taken 1839 times.
 2039 if (retval < 0) goto end;
7967
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1839 times.
 1839 if (tag != TAG_START) {
7968 ✗ retval = -1;
7969 ✗ goto end;
7970 }
7971
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1839 times.
 1839 if (x_type != x_version_reply) {
7972 ✗ retval = -1;
7973 ✗ goto end;
7974 }
7975
7976
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1839 times.
 1839 if (x_proto == x_unknown_proto) {
7977 ✗ G_DEBUG("no common protocol, returning error");
7978 ✗ retval = -1;
7979 ✗ goto end;
7980 }
7981
7982 /* This code will check if, in case of an upgrade if:
7983 - We are a node able to speak IPv6.
7984 - If we are connecting to a group that does not speak IPv6.
7985 - If our address is IPv4-compatible in order for the old group to be
7986 able to contact us back. */
7987
5/8
✓ Branch 0 taken 1812 times.
✓ Branch 1 taken 27 times.
✓ Branch 2 taken 1812 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 1812 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 1839 times.
 1839 if (is_cargo_type(a, add_node_type) && x_proto < minimum_ipv6_version() &&
7988 ✗ !are_we_allowed_to_upgrade_to_v6(a)) {
7989 ✗ retval = -1;
7990 ✗ goto end;
7991 }
7992
7993
4/6
✓ Branch 0 taken 1839 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 1838 times.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
 1839 G_DEBUG("client connection will use protocol version %d", x_proto);
7994 IFDBG(D_NONE, STRLIT("client connection will use protocol version ");
7995 NDBG(x_proto, u); STRLIT(xcom_proto_to_str(x_proto)));
7996 1839 fd->x_proto = x_proto;
7997 1839 set_connected(fd, CON_PROTO);
7998 }
7999 1842 msg->a = a;
8000 1842 msg->to = VOID_NODE_NO;
8001 1842 msg->op = client_msg;
8002 1842 msg->force_delivery = force;
8003
8004
1/2
✓ Branch 0 taken 1842 times.
✗ Branch 1 not taken.
 1842 serialized = serialize_msg(msg, fd->x_proto, &buflen, &buf);
8005
2/2
✓ Branch 0 taken 1839 times.
✓ Branch 1 taken 3 times.
 1842 if (serialized) {
8006
1/2
✓ Branch 0 taken 1839 times.
✗ Branch 1 not taken.
 1839 retval = socket_write(fd, buf, buflen);
8007 1839 if (buflen != retval) {
8008 IFDBG(D_NONE, FN; STRLIT("write failed "); NDBG(fd->fd, d);
8009 NDBG(buflen, d); NDBG64(retval));
8010 }
8011 } else {
8012 /* Failed to serialize, set retval accordingly. */
8013 3 retval = -1;
8014 }
8015 1842 X_FREE(buf);
8016 2042 end:
8017 2042 msg->a = nullptr; /* Do not deallocate a */
8018
1/2
✓ Branch 0 taken 2042 times.
✗ Branch 1 not taken.
 2042 XCOM_XDR_FREE(xdr_pax_msg, msg);
8019 2042 return retval;
8020 }
8021
8022 /* purecov: begin tested */
8023 /*
8024 * Tested by TEST_F(XComMultinodeSmokeTest,
8025 * 3_nodes_member_crashes_with_dieop_and_joins_again_immediately) GCS smoke
8026 * test
8027 */
8028 ✗ int64_t xcom_client_send_die(connection_descriptor *fd) {
8029 ✗ if (!fd) return 0;
8030 ✗ uint32_t buflen = 0;
8031 ✗ char *buf = nullptr;
8032 ✗ int64_t retval = 0;
8033 app_data a;
8034 ✗ pax_msg *msg = pax_msg_new(null_synode, nullptr);
8035
8036 ✗ if (!proto_done(fd)) {
8037 xcom_proto x_proto;
8038 x_msg_type x_type;
8039 unsigned int tag;
8040 ✗ retval = xcom_send_proto(fd, my_xcom_version, x_version_req, TAG_START);
8041 ✗ G_DEBUG("client sent negotiation request for protocol %d", my_xcom_version);
8042 ✗ if (retval < 0) goto end;
8043 ✗ retval = xcom_recv_proto(fd, &x_proto, &x_type, &tag);
8044 ✗ if (retval < 0) goto end;
8045 ✗ if (tag != TAG_START) {
8046 ✗ retval = -1;
8047 ✗ goto end;
8048 }
8049 ✗ if (x_type != x_version_reply) {
8050 ✗ retval = -1;
8051 ✗ goto end;
8052 }
8053
8054 ✗ if (x_proto == x_unknown_proto) {
8055 ✗ G_DEBUG("no common protocol, returning error");
8056 ✗ retval = -1;
8057 ✗ goto end;
8058 }
8059 ✗ G_DEBUG("client connection will use protocol version %d", x_proto);
8060 IFDBG(D_NONE, STRLIT("client connection will use protocol version ");
8061 NDBG(x_proto, u); STRLIT(xcom_proto_to_str(x_proto)));
8062 ✗ fd->x_proto = x_proto;
8063 ✗ set_connected(fd, CON_PROTO);
8064 }
8065 ✗ init_app_data(&a);
8066 ✗ a.body.c_t = app_type;
8067 ✗ msg->a = &a;
8068 ✗ msg->op = die_op;
8069 /*
8070 Set the msgno to a value that ensures the die_op will be processed by
8071 XCom when it is received (it needs to be higher than the msgno of the
8072 executed_msg, otherwise XCom will simply ignore it).
8073 */
8074 ✗ msg->synode.msgno = UINT64_MAX;
8075
8076 ✗ serialize_msg(msg, fd->x_proto, &buflen, &buf);
8077 ✗ if (buflen) {
8078 ✗ retval = socket_write(fd, buf, buflen);
8079 ✗ if (buflen != retval) {
8080 IFDBG(D_NONE, FN; STRLIT("write failed "); NDBG(fd->fd, d);
8081 NDBG(buflen, d); NDBG64(retval));
8082 }
8083 ✗ X_FREE(buf);
8084 }
8085 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8086 ✗ end:
8087 ✗ msg->a = nullptr;
8088 ✗ XCOM_XDR_FREE(xdr_pax_msg, msg);
8089 ✗ return retval > 0 && retval == buflen ? 1 : 0;
8090 }
8091 /* purecov: end */
8092
8093 #ifdef XCOM_STANDALONE
8094 /* purecov: begin deadcode */
8095 int64_t xcom_client_send_data(uint32_t size, char *data,
8096 connection_descriptor *fd) {
8097 if (!fd) return 0;
8098 app_data a;
8099 int64_t retval = 0;
8100 init_app_data(&a);
8101 a.body.c_t = app_type;
8102 a.body.app_u_u.data.data_len = size;
8103 a.body.app_u_u.data.data_val = data;
8104 retval = xcom_send_client_app_data(fd, &a, 0);
8105 xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8106 return retval;
8107 }
8108
8109 int64_t xcom_client_send_data_no_free(uint32_t size, char *data,
8110 connection_descriptor *fd) {
8111 if (!fd) return 0;
8112 app_data a;
8113 int64_t retval = 0;
8114 init_app_data(&a);
8115 a.body.c_t = app_type;
8116 a.body.app_u_u.data.data_len = size;
8117 a.body.app_u_u.data.data_val = data;
8118 retval = xcom_send_client_app_data(fd, &a, 0);
8119 return retval;
8120 }
8121 /* purecov: end */
8122 #endif
8123
8124 #ifndef _WIN32
8125 #include <arpa/inet.h>
8126 #include <netinet/in.h>
8127 #include <sys/socket.h>
8128 #endif
8129
8130 /* Output warning in log periodically if we receive messages
8131 with a protocol version that does not match our own */
8132 /* purecov: begin inspected */
8133 ✗ void warn_protoversion_mismatch(connection_descriptor *rfd) {
8134 struct sockaddr_storage sock_addr;
8135 ✗ socklen_t sock_size = sizeof(sock_addr);
8136
8137 ✗ if (task_now() - protoversion_warning_time > PROTOVERSION_WARNING_TIMEOUT) {
8138 ✗ if (0 ==
8139 ✗ xcom_getpeername(rfd->fd, (struct sockaddr *)&sock_addr, &sock_size)) {
8140 char buf[INET6_ADDRSTRLEN + 1];
8141 ✗ struct sockaddr_in *s4 = (struct sockaddr_in *)&sock_addr;
8142 ✗ struct sockaddr_in6 *s6 = (struct sockaddr_in6 *)&sock_addr;
8143 char const *ok;
8144
8145 ✗ memset((void *)buf, 0, sizeof(buf));
8146 ✗ if (sock_addr.ss_family == AF_INET) {
8147 ✗ ok = inet_ntop(sock_addr.ss_family, (void *)&s4->sin_addr, buf,
8148 sizeof(buf));
8149 } else {
8150 ✗ ok = inet_ntop(sock_addr.ss_family, (void *)&s6->sin6_addr, buf,
8151 sizeof(buf));
8152 }
8153 ✗ if (ok) {
8154 ✗ G_WARNING(
8155 "Detected incorrect xcom protocol version in connection from %s "
8156 "indicates missing cleanup of, or incorrect, xcom group "
8157 "definition on remote host. Please upgrade the process running on "
8158 "%s to a compatible version or stop it.",
8159 buf, buf);
8160 ✗ protoversion_warning_time = task_now();
8161 }
8162 }
8163 }
8164 }
8165 /* purecov: end */
8166
8167 1839 static pax_msg *socket_read_msg(connection_descriptor *rfd, pax_msg *p)
8168 /* Should buffer reads as well */
8169 {
8170 int64_t n;
8171 char *bytes;
8172 unsigned char header_buf[MSG_HDR_SIZE];
8173 xcom_proto x_version;
8174 uint32_t msgsize;
8175 x_msg_type x_type;
8176 unsigned int tag;
8177 1839 int deserialize_ok = 0;
8178
8179 1839 bytes = nullptr;
8180
8181 /* Read version, length, type, and tag */
8182
1/2
✓ Branch 0 taken 1839 times.
✗ Branch 1 not taken.
 1839 n = socket_read_bytes(rfd, (char *)header_buf, MSG_HDR_SIZE);
8183
8184
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1839 times.
 1839 if (n <= 0) {
8185 IFDBG(D_NONE, FN; NDBG64(n));
8186 ✗ return nullptr;
8187 }
8188
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1839 times.
 1839 assert(n == MSG_HDR_SIZE);
8189 1839 x_version = (xcom_proto)get_32(VERS_PTR(header_buf));
8190 /* Check the protocol version before doing anything else */
8191 #ifdef XCOM_PARANOID
8192 assert(check_protoversion(x_version, rfd->x_proto));
8193 #endif
8194
2/4
✓ Branch 0 taken 1839 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1839 times.
 1839 if (!check_protoversion(x_version, rfd->x_proto)) {
8195 /* purecov: begin inspected */
8196 ✗ warn_protoversion_mismatch(rfd);
8197 ✗ return nullptr;
8198 /* purecov: end */
8199 }
8200
8201 /* OK, we can grok this version */
8202
8203
1/2
✓ Branch 0 taken 1839 times.
✗ Branch 1 not taken.
 1839 get_header_1_0(header_buf, &msgsize, &x_type, &tag);
8204
8205 /* Allocate buffer space for message */
8206 1839 bytes = (char *)xcom_calloc(1, msgsize);
8207
8208 /* Read message */
8209
1/2
✓ Branch 0 taken 1839 times.
✗ Branch 1 not taken.
 1839 n = socket_read_bytes(rfd, bytes, msgsize);
8210
8211
1/2
✓ Branch 0 taken 1839 times.
✗ Branch 1 not taken.
 1839 if (n > 0) {
8212 /* Deserialize message */
8213
1/2
✓ Branch 0 taken 1839 times.
✗ Branch 1 not taken.
 1839 deserialize_ok = deserialize_msg(p, rfd->x_proto, bytes, msgsize);
8214 IFDBG(D_NONE, FN; STRLIT(" deserialized message"));
8215 }
8216 /* Deallocate buffer */
8217 1839 X_FREE(bytes);
8218
2/4
✓ Branch 0 taken 1839 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1839 times.
 1839 if (n <= 0 || deserialize_ok == 0) {
8219 IFDBG(D_NONE, FN; NDBG64(n));
8220 ✗ return nullptr;
8221 }
8222 1839 return (p);
8223 }
8224
8225 #ifdef XCOM_STANDALONE
8226 /* purecov: begin deadcode */
8227 int xcom_client_boot(connection_descriptor *fd, node_list *nl,
8228 uint32_t group_id) {
8229 if (!fd) return 0;
8230 app_data a;
8231 int retval = 0;
8232 retval = (int)xcom_send_client_app_data(
8233 fd, init_config_with_group(&a, nl, unified_boot_type, group_id), 0);
8234 xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8235 return retval;
8236 }
8237 /* purecov: end */
8238 #endif
8239
8240 enum xcom_send_app_wait_result {
8241 SEND_REQUEST_FAILED = 0,
8242 RECEIVE_REQUEST_FAILED,
8243 REQUEST_BOTCHED,
8244 RETRIES_EXCEEDED,
8245 REQUEST_OK_RECEIVED,
8246 REQUEST_FAIL_RECEIVED,
8247 REQUEST_OK_REDIRECT
8248 };
8249 typedef enum xcom_send_app_wait_result xcom_send_app_wait_result;
8250
8251 /**
8252 * Send a message and wait for response.
8253 *
8254 * The caller is reponsible for freeing p after calling this function,
8255 * i.e. xdr_free((xdrproc_t)xdr_pax_msg, (char *)p)
8256 */
8257 2042 static xcom_send_app_wait_result xcom_send_app_wait_and_get(
8258 connection_descriptor *fd, app_data *a, int force, pax_msg *p,
8259 leader_info_data *leaders) {
8260 2042 int retval = 0;
8261 2042 int retry_count = 10; /* Same as 'connection_attempts' */
8262 2042 pax_msg *rp = nullptr;
8263
8264 do {
8265 2042 retval = (int)xcom_send_client_app_data(fd, a, force);
8266 2042 memset(p, 0, sizeof(*p)); /* before return so caller can free p */
8267
2/2
✓ Branch 0 taken 203 times.
✓ Branch 1 taken 1839 times.
 2042 if (retval < 0) {
8268 203 return SEND_REQUEST_FAILED;
8269 }
8270 1839 rp = socket_read_msg(fd, p);
8271
1/2
✓ Branch 0 taken 1839 times.
✗ Branch 1 not taken.
 1839 if (rp) {
8272 1839 client_reply_code cli_err = rp->cli_err;
8273
2/5
✓ Branch 0 taken 1360 times.
✓ Branch 1 taken 479 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 1839 switch (cli_err) {
8274 1360 case REQUEST_OK:
8275 1360 return REQUEST_OK_RECEIVED;
8276 479 case REQUEST_FAIL:
8277
2/4
✓ Branch 0 taken 479 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 479 times.
✗ Branch 3 not taken.
 479 G_INFO(
8278 "Sending a request to a remote XCom failed. Please check the "
8279 "remote node log for more details.")
8280 479 return REQUEST_FAIL_RECEIVED;
8281 ✗ case REQUEST_RETRY:
8282 ✗ if (retry_count > 1) xdr_free((xdrproc_t)xdr_pax_msg, (char *)p);
8283 ✗ G_INFO(
8284 "Retrying a request to a remote XCom. Please check the remote "
8285 "node log for more details.")
8286 ✗ xcom_sleep(1);
8287 ✗ break;
8288 ✗ case REQUEST_REDIRECT:
8289 ✗ G_DEBUG("cli_err %d", cli_err);
8290 ✗ if (leaders && rp->rd && rp->rd->rt == leader_info) {
8291 ✗ *leaders = steal_leader_info_data(rp->rd->reply_data_u.leaders);
8292 }
8293 ✗ xdr_free((xdrproc_t)xdr_pax_msg, (char *)p);
8294 ✗ return REQUEST_OK_REDIRECT;
8295 ✗ default:
8296 ✗ G_WARNING("XCom client connection has received an unknown response.");
8297 ✗ return REQUEST_BOTCHED;
8298 }
8299 } else {
8300 ✗ G_WARNING("Reading a request from a remote XCom failed.");
8301 ✗ return RECEIVE_REQUEST_FAILED;
8302 }
8303 ✗ } while (--retry_count);
8304 /* Timeout after REQUEST_RETRY has been received 'retry_count' times */
8305 ✗ G_MESSAGE(
8306 "Request failed: maximum number of retries (10) has been exhausted.");
8307 ✗ return RETRIES_EXCEEDED;
8308 }
8309
8310 2039 static int xcom_send_app_wait(connection_descriptor *fd, app_data *a, int force,
8311 leader_info_data *leaders) {
8312 pax_msg p;
8313 2039 int result = 0;
8314 2039 memset(&p, 0, sizeof(p));
8315 xcom_send_app_wait_result res =
8316
1/2
✓ Branch 0 taken 2039 times.
✗ Branch 1 not taken.
 2039 xcom_send_app_wait_and_get(fd, a, force, &p, leaders);
8317
2/3
✓ Branch 0 taken 679 times.
✓ Branch 1 taken 1360 times.
✗ Branch 2 not taken.
 2039 switch (res) {
8318 679 case SEND_REQUEST_FAILED:
8319 case RECEIVE_REQUEST_FAILED:
8320 case REQUEST_BOTCHED:
8321 case RETRIES_EXCEEDED:
8322 case REQUEST_FAIL_RECEIVED:
8323 case REQUEST_OK_REDIRECT:
8324 679 result = 0;
8325 679 break;
8326 1360 case REQUEST_OK_RECEIVED:
8327 1360 result = 1;
8328 1360 break;
8329 }
8330
1/2
✓ Branch 0 taken 2039 times.
✗ Branch 1 not taken.
 2039 xdr_free((xdrproc_t)xdr_pax_msg, (char *)&p);
8331 2039 return result;
8332 }
8333
8334 2039 int xcom_send_cfg_wait(connection_descriptor *fd, node_list *nl,
8335 uint32_t group_id, cargo_type ct, int force) {
8336 app_data a;
8337 2039 int retval = 0;
8338 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(nl)););
8339
2/4
✓ Branch 0 taken 2039 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2039 times.
✗ Branch 3 not taken.
 2039 retval = xcom_send_app_wait(fd, init_config_with_group(&a, nl, ct, group_id),
8340 force, nullptr);
8341
1/2
✓ Branch 0 taken 2039 times.
✗ Branch 1 not taken.
 2039 xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8342 2039 return retval;
8343 }
8344
8345 2012 int xcom_client_add_node(connection_descriptor *fd, node_list *nl,
8346 uint32_t group_id) {
8347
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2012 times.
 2012 if (!fd) return 0;
8348 u_int i;
8349
2/2
✓ Branch 0 taken 2012 times.
✓ Branch 1 taken 2012 times.
 4024 for (i = 0; i < nl->node_list_len; i++) {
8350
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2012 times.
 2012 assert(nl->node_list_val[i].proto.max_proto > x_unknown_proto);
8351 }
8352 2012 return xcom_send_cfg_wait(fd, nl, group_id, add_node_type, 0);
8353 }
8354
8355 27 int xcom_client_remove_node(connection_descriptor *fd, node_list *nl,
8356 uint32_t group_id) {
8357
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 27 times.
 27 if (!fd) return 0;
8358 27 return xcom_send_cfg_wait(fd, nl, group_id, remove_node_type, 0);
8359 }
8360
8361 ✗ static int xcom_check_reply(int const res) {
8362 ✗ return res == REQUEST_OK_RECEIVED;
8363 }
8364
8365 #if 0
8366 /* purecov: begin deadcode */
8367 int xcom_client_get_event_horizon(connection_descriptor *fd, uint32_t group_id,
8368 xcom_event_horizon *event_horizon) {
8369 if (!fd) return 0;
8370 pax_msg p;
8371 app_data a;
8372 int result = 0;
8373
8374 memset(&p, 0, sizeof(p));
8375
8376 xcom_send_app_wait_result res = xcom_send_app_wait_and_get(
8377 fd, init_get_event_horizon_msg(&a, group_id), 0, &p, 0);
8378 result = xcom_check_reply(res);
8379 if (result) {
8380 *event_horizon = p.event_horizon;
8381 }
8382
8383 xdr_free((xdrproc_t)xdr_pax_msg, (char *)&p);
8384 xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8385
8386 return result;
8387 }
8388 /* purecov: end */
8389
8390 /* purecov: begin deadcode */
8391 int xcom_client_set_event_horizon(connection_descriptor *fd, uint32_t group_id,
8392 xcom_event_horizon event_horizon) {
8393 if (!fd) return 0;
8394 app_data a;
8395 int retval = 0;
8396 retval = xcom_send_app_wait(
8397 fd, init_set_event_horizon_msg(&a, group_id, event_horizon), 0, 0);
8398 xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8399 return retval;
8400 }
8401 /* purecov: end */
8402 #endif
8403
8404 3 int xcom_client_get_synode_app_data(connection_descriptor *const fd,
8405 uint32_t group_id,
8406 synode_no_array *const synodes,
8407 synode_app_data_array *const reply) {
8408
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3 times.
 3 if (!fd) return 0;
8409 3 bool_t const success = TRUE;
8410 3 bool_t const failure = FALSE;
8411 3 bool_t result = failure;
8412 pax_msg p;
8413 app_data a;
8414 3 u_int const nr_synodes_requested = synodes->synode_no_array_len;
8415
8416 /* This call moves, as in C++ move semantics, synodes into app_data a. */
8417
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 init_get_synode_app_data_msg(&a, group_id, synodes);
8418
8419 {
8420 xcom_send_app_wait_result res =
8421
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 xcom_send_app_wait_and_get(fd, &a, 0, &p, nullptr);
8422
1/3
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
 3 switch (res) {
8423 3 case RECEIVE_REQUEST_FAILED:
8424 case REQUEST_BOTCHED:
8425 case RETRIES_EXCEEDED:
8426 case SEND_REQUEST_FAILED:
8427 case REQUEST_FAIL_RECEIVED:
8428 case REQUEST_OK_REDIRECT: {
8429
3/6
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 3 times.
✗ Branch 5 not taken.
 3 G_TRACE(
8430 "xcom_client_get_synode_app_data: XCom did not have the required "
8431 "%u "
8432 "synodes.",
8433 nr_synodes_requested);
8434 3 break;
8435 }
8436 ✗ case REQUEST_OK_RECEIVED: {
8437 ✗ u_int const nr_synodes_received =
8438 p.requested_synode_app_data.synode_app_data_array_len;
8439 ✗ G_TRACE(
8440 "xcom_client_get_synode_app_data: Got %u synode payloads, we "
8441 "asked "
8442 "for %u.",
8443 nr_synodes_received, nr_synodes_requested);
8444
8445 /* This should always be TRUE.
8446 * But rather than asserting it, let's treat an unexpected number of
8447 * synode payloads in the reply as a failure. */
8448 ✗ if (nr_synodes_received == nr_synodes_requested) {
8449 /* Move (as in C++ move semantics) into reply */
8450 ✗ synode_app_data_array_move(reply, &p.requested_synode_app_data);
8451 ✗ result = success;
8452 }
8453 ✗ break;
8454 }
8455 }
8456 }
8457
8458
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 xdr_free((xdrproc_t)xdr_pax_msg, (char *)&p);
8459
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8460
8461 3 return result;
8462 }
8463
8464 #ifdef NOTDEF
8465 /* Not completely implemented, need to be handled properly
8466 when received as a client message in dispatch_op.
8467 Should have separate opcode from normal add/remove,
8468 like force config_type */
8469 int xcom_client_force_add_node(connection_descriptor *fd, node_list *nl,
8470 uint32_t group_id) {
8471 if (!fd) return 0;
8472 return xcom_send_cfg_wait(fd, nl, group_id, add_node_type, 1);
8473 }
8474
8475 int xcom_client_force_remove_node(connection_descriptor *fd, node_list *nl,
8476 uint32_t group_id) {
8477 if (!fd) return 0;
8478 return xcom_send_cfg_wait(fd, nl, group_id, remove_node_type, 1);
8479 }
8480 #endif
8481
8482 /* purecov: begin deadcode */
8483 ✗ int xcom_client_enable_arbitrator(connection_descriptor *fd) {
8484 ✗ if (!fd) return 0;
8485 app_data a;
8486 ✗ int retval = 0;
8487 ✗ init_app_data(&a);
8488 ✗ a.body.c_t = enable_arbitrator;
8489 ✗ retval = xcom_send_app_wait(fd, &a, 0, nullptr);
8490 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8491 ✗ return retval;
8492 }
8493 /* purecov: end */
8494
8495 /* purecov: begin deadcode */
8496 ✗ int xcom_client_disable_arbitrator(connection_descriptor *fd) {
8497 ✗ if (!fd) return 0;
8498 app_data a;
8499 ✗ int retval = 0;
8500 ✗ init_app_data(&a);
8501 ✗ a.body.c_t = disable_arbitrator;
8502 ✗ retval = xcom_send_app_wait(fd, &a, 0, nullptr);
8503 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8504 ✗ return retval;
8505 }
8506 /* purecov: end */
8507
8508 /* purecov: begin deadcode */
8509 ✗ int xcom_client_set_cache_limit(connection_descriptor *fd,
8510 uint64_t cache_limit) {
8511 ✗ if (!fd) return 0;
8512 app_data a;
8513 ✗ int retval = 0;
8514 ✗ init_app_data(&a);
8515 ✗ a.body.c_t = set_cache_limit;
8516 ✗ a.body.app_u_u.cache_limit = cache_limit;
8517 ✗ retval = xcom_send_app_wait(fd, &a, 0, nullptr);
8518 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8519 ✗ return retval;
8520 }
8521 /* purecov: end */
8522
8523 ✗ int xcom_client_convert_into_local_server(connection_descriptor *const fd) {
8524 ✗ if (!fd) return 0;
8525 app_data a;
8526 ✗ int retval = 0;
8527 ✗ retval = xcom_send_app_wait(fd, init_convert_into_local_server_msg(&a), 0,
8528 nullptr);
8529 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8530 ✗ return retval;
8531 }
8532
8533 /* Set max number of leaders */
8534 93 void init_set_max_leaders(uint32_t group_id, app_data *a, node_no max_leaders) {
8535 93 init_app_data(a);
8536 93 a->app_key.group_id = a->group_id = group_id;
8537 93 a->body.c_t = set_max_leaders;
8538 93 a->body.app_u_u.max_leaders = max_leaders;
8539 93 }
8540
8541 /* Set max number of leaders */
8542 12 int xcom_client_set_max_leaders(connection_descriptor *fd, node_no max_leaders,
8543 uint32_t group_id) {
8544
1/2
✓ Branch 0 taken 12 times.
✗ Branch 1 not taken.
 12 if (!fd) return 0;
8545 app_data a;
8546 ✗ int retval = 0;
8547 ✗ init_set_max_leaders(group_id, &a, max_leaders);
8548 ✗ retval = xcom_send_app_wait(fd, &a, 0, nullptr);
8549 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8550 ✗ return retval;
8551 }
8552
8553 87 leader_array new_leader_array(u_int n, char const *names[]) {
8554
1/2
✓ Branch 0 taken 87 times.
✗ Branch 1 not taken.
 87 leader_array leaders = alloc_leader_array(n);
8555
2/2
✓ Branch 0 taken 79 times.
✓ Branch 1 taken 87 times.
 166 for (u_int i = 0; i < n; i++) {
8556 79 leaders.leader_array_val[i].address = strdup(names[i]);
8557 }
8558 87 return leaders;
8559 }
8560
8561 /* Set new set of active leaders. Does not deallocate leaders. */
8562 87 void init_set_leaders(uint32_t group_id, app_data *a,
8563 leader_array const leaders) {
8564 87 init_app_data(a);
8565 87 a->app_key.group_id = a->group_id = group_id;
8566 87 a->body.c_t = set_leaders_type;
8567 // We could have avoided this copy, but having leaders as const
8568 // makes it easier to reason about sharing
8569 87 a->body.app_u_u.leaders = clone_leader_array(leaders);
8570 87 }
8571
8572 /* Set new set of active leaders. */
8573 12 void init_set_leaders(uint32_t group_id, app_data *a, u_int n,
8574 char const *names[]) {
8575
1/2
✓ Branch 0 taken 12 times.
✗ Branch 1 not taken.
 12 leader_array leaders = new_leader_array(n, names);
8576
1/2
✓ Branch 0 taken 12 times.
✗ Branch 1 not taken.
 12 init_set_leaders(group_id, a, leaders);
8577 // leaders have been copied, so deallocate
8578
1/2
✓ Branch 0 taken 12 times.
✗ Branch 1 not taken.
 12 xdr_free((xdrproc_t)xdr_leader_array, (char *)(&leaders));
8579 12 }
8580
8581 75 void init_set_leaders(uint32_t group_id, app_data *leader_app,
8582 leader_array const leaders, app_data *max_app,
8583 node_no max_leaders) {
8584 75 init_set_leaders(group_id, leader_app, leaders);
8585 75 init_set_max_leaders(group_id, max_app, max_leaders);
8586 75 leader_app->next = max_app;
8587 75 }
8588
8589 75 void init_set_leaders(uint32_t group_id, app_data *leader_app, u_int n,
8590 char const *names[], app_data *max_app,
8591 node_no max_leaders) {
8592
1/2
✓ Branch 0 taken 75 times.
✗ Branch 1 not taken.
 75 leader_array leaders = new_leader_array(n, names);
8593
1/2
✓ Branch 0 taken 75 times.
✗ Branch 1 not taken.
 75 init_set_leaders(group_id, leader_app, leaders, max_app, max_leaders);
8594 // leaders have been copied, so deallocate
8595
1/2
✓ Branch 0 taken 75 times.
✗ Branch 1 not taken.
 75 xdr_free((xdrproc_t)xdr_leader_array, (char *)(&leaders));
8596 75 }
8597
8598 /* Set new set of active leaders. */
8599 15 int xcom_client_set_leaders(connection_descriptor *fd, u_int n,
8600 char const *names[], uint32_t group_id) {
8601
1/2
✓ Branch 0 taken 15 times.
✗ Branch 1 not taken.
 15 if (!fd) return 0;
8602 app_data a;
8603 ✗ init_set_leaders(group_id, &a, n, names);
8604 ✗ int retval = xcom_send_app_wait(fd, &a, 0, nullptr);
8605 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8606 ✗ return retval;
8607 }
8608
8609 std::unique_ptr<Network_provider_management_interface>
8610 58393 get_network_management_interface() {
8611 58393 return std::make_unique<Network_Management_Interface>();
8612 }
8613
8614 std::unique_ptr<Network_provider_operations_interface>
8615 2365 get_network_operations_interface() {
8616 2365 return std::make_unique<Network_Management_Interface>();
8617 }
8618
8619 /* Set new set of active leaders and number of leaders. */
8620 ✗ int xcom_client_set_leaders(connection_descriptor *fd, u_int n,
8621 char const *names[], node_no max_leaders,
8622 uint32_t group_id) {
8623 ✗ if (!fd) return 0;
8624 app_data leader_app;
8625 app_data max_app;
8626 ✗ int retval = 0;
8627 ✗ init_set_leaders(group_id, &leader_app, n, names, &max_app, max_leaders);
8628 ✗ retval = xcom_send_app_wait(fd, &leader_app, 0, nullptr);
8629 // leader_app and max_app have been linked, so unlink
8630 // to avoid deallocating the stack objects.
8631 ✗ leader_app.next = nullptr;
8632 ✗ max_app.next = nullptr;
8633 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&leader_app);
8634 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&max_app);
8635 ✗ return retval;
8636 }
8637
8638 ✗ int xcom_client_get_leaders(connection_descriptor *fd, uint32_t group_id,
8639 leader_info_data *leaders) {
8640 ✗ if (!fd) return 0;
8641 pax_msg p;
8642 app_data a;
8643 ✗ int result = 0;
8644
8645 ✗ memset(&p, 0, sizeof(p));
8646
8647 ✗ xcom_send_app_wait_result res = xcom_send_app_wait_and_get(
8648 fd, init_get_msg(&a, group_id, get_leaders_type), 0, &p, nullptr);
8649 ✗ result = xcom_check_reply(res);
8650 ✗ if (result) {
8651 // Steal the returned data
8652 ✗ *leaders = steal_leader_info_data(p.rd->reply_data_u.leaders);
8653 }
8654
8655 ✗ xdr_free((xdrproc_t)xdr_pax_msg, (char *)&p);
8656 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8657
8658 ✗ return result;
8659 }
8660
8661 #if 0
8662 /* Called when leader changes */
8663 static xcom_election_cb election_cb [[maybe_unused]] = NULL;
8664
8665 void set_xcom_election_cb(xcom_election_cb x) { election_cb = x; }
8666 #endif
8667
8668 // The timer code and the associated Paxos FSM stuff is only used for
8669 // tracking/debugging Paxos state transitions at the moment, but I believe the
8670 // FSM is correct, and if used for actually handling the incoming messages,
8671 // would make the code simpler, and easier to understand and reason about by
8672 // making lots of tests scattered around in the code unnecessary. I have had
8673 // this on the backburner for a long time, and used the single writer worklog
8674 // to test it, but did not dare to actually replace the existing logic in the
8675 // scope of this worklog.Take it as a hint for future improvement of the code
8676 // base...
8677
8678 // The time queue as configured now will allow up to 10 seconds delay with
8679 // TICK_PERIOD (0.01) seconds granularity. All machines which map to the same
8680 // time slot will wake up simultaneously. The complexity when inserting or
8681 // removing a pax_machine is O(1), but this is somewhat offset by the need to
8682 // advance the current tick for every TICK_PERIOD. Not a problem in practice,
8683 // and the code is dead simple.
8684
8685 /* Max number of ticks before wrapping. With 10 ms per step, this will give a
8686 * max delay of 10 seconds, which is plenty for the Paxos timers */
8687
8688 enum { paxos_timer_range = 1000 };
8689 /* Ten milliseconds granularity is sufficient */
8690 #define TICK_PERIOD 0.01
8691
8692 /* The index into the time queue */
8693 static unsigned int current_tick = 0;
8694
8695 /* The time queue is an array of timers. Each timer is the head of
8696 a possibly empty list of timers */
8697
8698 static linkage time_queue[paxos_timer_range];
8699
8700 2355 static void init_time_queue() {
8701 unsigned int i;
8702
2/2
✓ Branch 0 taken 2355000 times.
✓ Branch 1 taken 2355 times.
 2357355 for (i = 0; i < paxos_timer_range; i++) {
8703 2355000 link_init(&time_queue[i], TYPE_HASH("time_queue"));
8704 }
8705 2355 }
8706
8707 /* Put pax_machine into the time queue at the correct place */
8708 261128 static void paxos_twait(pax_machine *p, unsigned int t) {
8709 /* Guard aginst 0 delay, which would become max delay */
8710
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 261128 times.
 261128 if (0 == t) t = 1;
8711 261128 unsigned int pos = (current_tick + t) % paxos_timer_range;
8712 261128 link_into(&p->watchdog, &time_queue[pos]);
8713
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 261128 times.
 261128 assert(!link_empty(&time_queue[pos]));
8714 261128 }
8715
8716 /* Remove pax_machine from timer queue */
8717 599444 static void paxos_twait_cancel(pax_machine *p) { link_out(&p->watchdog); }
8718
8719 /* Wake all pax_machines waiting at time slot t */
8720 8326948 static void paxos_wakeup(unsigned int t) {
8721 8326948 linkage *head = &time_queue[t];
8722 linkage *p;
8723 8326948 if (!link_empty(head)) {
8724 IFDBG(D_CONS, FN; NUMEXP(t); NUMEXP(link_empty(head)));
8725 }
8726
2/2
✓ Branch 0 taken 3760 times.
✓ Branch 1 taken 8326948 times.
 8330708 while (!link_empty(head)) {
8727 3760 p = link_first(head);
8728 3760 paxos_timeout(container_of(p, pax_machine, watchdog));
8729 3760 link_out(p);
8730 }
8731 8326948 }
8732
8733 /* Advance current_tick to next slot and wake all pax_machines there */
8734 8326948 static void paxos_timer_advance() {
8735 8326948 current_tick = (current_tick + 1) % paxos_timer_range;
8736 8326948 paxos_wakeup(current_tick);
8737 8326948 }
8738
8739 /* Fire any expired timer for a Paxos machine */
8740 8331644 static int paxos_timer_task(task_arg arg [[maybe_unused]]) {
8741 DECL_ENV
8742 double start;
8743 2355 ENV_INIT
8744 2355 END_ENV_INIT
8745 END_ENV;
8746
6/9
✓ Branch 0 taken 2355 times.
✓ Branch 1 taken 8329289 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2355 times.
✓ Branch 5 taken 2355 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 7 times.
✓ Branch 8 taken 2348 times.
 8331644 TASK_BEGIN
8747 2348 ep->start = task_now();
8748
1/2
✓ Branch 0 taken 8329296 times.
✗ Branch 1 not taken.
 8329296 while (!xcom_shutdown) {
8749 8329296 ep->start += TICK_PERIOD;
8750
4/6
✗ Branch 0 not taken.
✓ Branch 1 taken 8329289 times.
✓ Branch 2 taken 8329289 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2341 times.
✓ Branch 5 taken 8326948 times.
 16658585 TASK_DELAY_UNTIL(ep->start);
8751 8326948 paxos_timer_advance();
8752 }
8753 ✗ FINALLY
8754 IFDBG(D_CONS, FN; STRLIT(" shutdown "));
8755
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2348 times.
 2348 TASK_END;
8756 }
8757
8758 /* The state functions/thunks */
8759 static int paxos_fsm_p1_master_enter(pax_machine *paxos, site_def const *site,
8760 paxos_event event, pax_msg *mess);
8761
8762 static int paxos_fsm_p1_master_wait(pax_machine *paxos, site_def const *site,
8763 paxos_event event, pax_msg *mess);
8764
8765 static int paxos_fsm_p2_master_enter(pax_machine *paxos, site_def const *site,
8766 paxos_event event, pax_msg *mess);
8767
8768 static int paxos_fsm_p2_master_wait(pax_machine *paxos, site_def const *site,
8769 paxos_event event, pax_msg *mess);
8770
8771 static int paxos_fsm_p2_slave_enter(pax_machine *paxos, site_def const *site,
8772 paxos_event event, pax_msg *mess);
8773
8774 static int paxos_fsm_p2_slave_wait(pax_machine *paxos, site_def const *site,
8775 paxos_event event, pax_msg *mess);
8776
8777 static int paxos_fsm_p3_master_wait(pax_machine *paxos, site_def const *site,
8778 paxos_event event, pax_msg *mess);
8779
8780 static int paxos_fsm_p3_slave_enter(pax_machine *paxos, site_def const *site,
8781 paxos_event event, pax_msg *mess);
8782
8783 static int paxos_fsm_p3_slave_wait(pax_machine *paxos, site_def const *site,
8784 paxos_event event, pax_msg *mess);
8785
8786 static int paxos_fsm_finished(pax_machine *paxos, site_def const *site,
8787 paxos_event event, pax_msg *mess);
8788
8789 typedef void (*paxos_state_action)(pax_machine *paxos, site_def const *site,
8790 pax_msg *mess);
8791
8792 8912 static int accept_new_prepare(pax_machine *paxos, pax_msg *mess) {
8793
1/2
✓ Branch 0 taken 8912 times.
✗ Branch 1 not taken.
 17824 return noop_match(paxos, mess) ||
8794
2/2
✓ Branch 0 taken 8888 times.
✓ Branch 1 taken 24 times.
 17824 gt_ballot(mess->proposal, paxos->acceptor.promise);
8795 }
8796
8797 10975 static int accept_new_accept(pax_machine *paxos, pax_msg *mess) {
8798
2/2
✓ Branch 0 taken 10772 times.
✓ Branch 1 taken 203 times.
 21747 return noop_match(paxos, mess) ||
8799
1/2
✓ Branch 0 taken 10772 times.
✗ Branch 1 not taken.
 21747 !gt_ballot(paxos->acceptor.promise, mess->proposal);
8800 }
8801
8802 261128 static int own_message(pax_msg *mess, site_def const *site) {
8803 261128 return is_local_node(mess->from, site);
8804 }
8805
8806 // Default paxos timeout in ticks
8807 // Change this if the FSM is used for anything else than debugging
8808 unsigned int constexpr const paxos_default_timeout = 100;
8809
8810 /* You are in a maze of little twisting functions ... */
8811
8812 27437 static void action_paxos_prepare(pax_machine *paxos, site_def const *site,
8813 pax_msg *mess) {
8814
2/2
✓ Branch 0 taken 16952 times.
✓ Branch 1 taken 10485 times.
 27437 if (own_message(mess, site)) {
8815 /* Wait for ack_prepare */
8816 16952 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p1_master_wait);
8817 } else {
8818 /* Wait for accept */
8819 10485 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p2_slave_enter);
8820 }
8821 27437 paxos_twait(paxos, paxos_default_timeout);
8822 27437 }
8823
8824 213828 static void action_paxos_accept(pax_machine *paxos, site_def const *site,
8825 pax_msg *mess) {
8826
2/2
✓ Branch 0 taken 108513 times.
✓ Branch 1 taken 105315 times.
 213828 if (own_message(mess, site)) {
8827 /* Wait for ack_accept */
8828 108513 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p2_master_wait);
8829 } else {
8830 /* Wait for learn */
8831 105315 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p3_slave_enter);
8832 }
8833 213828 paxos_twait(paxos, paxos_default_timeout);
8834 213828 }
8835
8836 599444 static void action_paxos_learn(pax_machine *paxos, site_def const *site,
8837 pax_msg *mess) {
8838 (void)site;
8839 (void)mess;
8840 /* We are finished */
8841 599444 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_finished);
8842 599444 paxos_twait_cancel(paxos);
8843 599444 }
8844
8845 ✗ static void action_paxos_start(pax_machine *paxos, site_def const *site,
8846 pax_msg *mess) {
8847 (void)site;
8848 (void)mess;
8849 /* Find value of this instance */
8850 ✗ SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p1_master_enter);
8851 ✗ paxos_twait(paxos, paxos_default_timeout);
8852 }
8853
8854 8912 static void action_new_prepare(pax_machine *paxos, site_def const *site,
8855 pax_msg *mess) {
8856 (void)site;
8857
2/2
✓ Branch 0 taken 8888 times.
✓ Branch 1 taken 24 times.
 8912 if (accept_new_prepare(paxos, mess)) {
8858 /* Wait for accept */
8859
2/2
✓ Branch 0 taken 8420 times.
✓ Branch 1 taken 468 times.
 8888 if (own_message(mess, site)) {
8860 /* Wait for ack_prepare */
8861 8420 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p1_master_enter);
8862 } else {
8863 /* Wait for accept */
8864 468 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p2_slave_enter);
8865 }
8866 8888 paxos_twait(paxos, paxos_default_timeout);
8867 }
8868 8912 }
8869
8870 26602 static void action_ack_prepare(pax_machine *paxos, site_def const *site,
8871 pax_msg *mess) {
8872 (void)mess;
8873
2/2
✓ Branch 0 taken 10232 times.
✓ Branch 1 taken 16370 times.
 26602 if (check_propose(site, paxos)) {
8874 /* Wait for accept */
8875 10232 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p2_master_enter);
8876 }
8877 26602 }
8878
8879 10975 static void action_new_accept(pax_machine *paxos, site_def const *site,
8880 pax_msg *mess) {
8881
1/2
✓ Branch 0 taken 10975 times.
✗ Branch 1 not taken.
 10975 if (accept_new_accept(paxos, mess)) {
8882 /* Wait for accept */
8883
2/2
✓ Branch 0 taken 543 times.
✓ Branch 1 taken 10432 times.
 10975 if (own_message(mess, site)) {
8884 /* Wait for ack_accept */
8885 543 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p2_master_enter);
8886 } else {
8887 /* Wait for learn */
8888 10432 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p3_slave_enter);
8889 }
8890 10975 paxos_twait(paxos, paxos_default_timeout);
8891 }
8892 10975 }
8893
8894 211845 static void action_ack_accept(pax_machine *paxos, site_def const *site,
8895 pax_msg *mess) {
8896 (void)mess;
8897
2/2
✓ Branch 0 taken 118014 times.
✓ Branch 1 taken 93831 times.
 211845 if (learn_ok(site, paxos)) {
8898 /* Wait for learn message */
8899 118014 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p3_master_wait);
8900 }
8901 211845 }
8902
8903 7300 static void action_ignorant(pax_machine *paxos, site_def const *site,
8904 pax_msg *mess) {
8905 (void)paxos;
8906 (void)site;
8907 (void)mess;
8908 7300 }
8909
8910 /* Dispatch tables for each state */
8911 paxos_state_action p1_idle_vtbl[last_p_event] = {action_paxos_prepare,
8912 nullptr,
8913 action_paxos_accept,
8914 nullptr,
8915 action_paxos_learn,
8916 action_paxos_start,
8917 nullptr};
8918 paxos_state_action p1_master_enter_vtbl[last_p_event] = {action_new_prepare,
8919 action_ack_prepare,
8920 action_new_accept,
8921 nullptr,
8922 action_paxos_learn,
8923 nullptr,
8924 nullptr};
8925 paxos_state_action p1_master_wait_vtbl[last_p_event] = {action_new_prepare,
8926 action_ack_prepare,
8927 action_new_accept,
8928 nullptr,
8929 action_paxos_learn,
8930 nullptr,
8931 nullptr};
8932 paxos_state_action p2_master_enter_vtbl[last_p_event] = {action_new_accept,
8933 nullptr,
8934 action_new_accept,
8935 action_ack_accept,
8936 action_paxos_learn,
8937 nullptr,
8938 nullptr};
8939 paxos_state_action p2_master_wait_vtbl[last_p_event] = {action_new_prepare,
8940 nullptr,
8941 action_new_accept,
8942 action_ack_accept,
8943 action_paxos_learn,
8944 nullptr,
8945 nullptr};
8946 paxos_state_action p2_slave_wait_vtbl[last_p_event] = {action_new_prepare,
8947 nullptr,
8948 action_new_accept,
8949 nullptr,
8950 action_paxos_learn,
8951 nullptr,
8952 nullptr};
8953 paxos_state_action p3_master_wait_vtbl[last_p_event] = {action_new_prepare,
8954 nullptr,
8955 action_new_accept,
8956 nullptr,
8957 action_paxos_learn,
8958 nullptr,
8959 nullptr};
8960 paxos_state_action p3_slave_wait_vtbl[last_p_event] = {action_new_prepare,
8961 nullptr,
8962 action_new_accept,
8963 nullptr,
8964 action_paxos_learn,
8965 nullptr,
8966 nullptr};
8967 paxos_state_action p_finished_vtbl[last_p_event] = {
8968 action_ignorant, nullptr, action_ignorant, nullptr,
8969 nullptr, nullptr, nullptr};
8970
8971 1284480 static inline void dispatch_p_event(paxos_state_action *vtbl,
8972 pax_machine *paxos, site_def const *site,
8973 paxos_event event, pax_msg *mess) {
8974
2/2
✓ Branch 0 taken 1106343 times.
✓ Branch 1 taken 178137 times.
 1284480 if (vtbl[event]) {
8975 1106343 vtbl[event](paxos, site, mess);
8976 }
8977 1284480 }
8978
8979 /* init state */
8980 599655 int paxos_fsm_idle(pax_machine *paxos, site_def const *site, paxos_event event,
8981 pax_msg *mess) {
8982 IFDBG(D_CONS, FN;);
8983 599655 dispatch_p_event(p1_idle_vtbl, paxos, site, event, mess);
8984 599655 return 0;
8985 }
8986
8987 /* Phase 1 master enter */
8988 8420 static int paxos_fsm_p1_master_enter(pax_machine *paxos, site_def const *site,
8989 paxos_event event, pax_msg *mess) {
8990 (void)site;
8991 (void)event;
8992 (void)mess;
8993 IFDBG(D_CONS, FN;);
8994 /* Send prepare and start timer */
8995 8420 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p1_master_wait);
8996 8420 return 0;
8997 }
8998
8999 /* Phase 1 master wait */
9000 41580 static int paxos_fsm_p1_master_wait(pax_machine *paxos, site_def const *site,
9001 paxos_event event, pax_msg *mess) {
9002 IFDBG(D_CONS, FN;);
9003 41580 dispatch_p_event(p1_master_wait_vtbl, paxos, site, event, mess);
9004 41580 return 0;
9005 }
9006
9007 /* Phase 2 master enter */
9008 10775 static int paxos_fsm_p2_master_enter(pax_machine *paxos, site_def const *site,
9009 paxos_event event, pax_msg *mess) {
9010 (void)site;
9011 (void)event;
9012 (void)mess;
9013 IFDBG(D_CONS, FN;);
9014 /* Send prepare and start timer */
9015 10775 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p2_master_wait);
9016 10775 return 0;
9017 }
9018
9019 /* Phase 2 master wait */
9020 213163 static int paxos_fsm_p2_master_wait(pax_machine *paxos, site_def const *site,
9021 paxos_event event, pax_msg *mess) {
9022 IFDBG(D_CONS, FN;);
9023 213163 dispatch_p_event(p2_master_wait_vtbl, paxos, site, event, mess);
9024 213163 return 0;
9025 }
9026
9027 /* Phase 2 slave enter */
9028 10943 static int paxos_fsm_p2_slave_enter(pax_machine *paxos, site_def const *site,
9029 paxos_event event, pax_msg *mess) {
9030 (void)site;
9031 (void)event;
9032 (void)mess;
9033 IFDBG(D_CONS, FN;);
9034 /* Start timer */
9035 10943 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p2_slave_wait);
9036 10943 return 1;
9037 }
9038
9039 /* Phase 2 slave wait */
9040 10943 static int paxos_fsm_p2_slave_wait(pax_machine *paxos, site_def const *site,
9041 paxos_event event, pax_msg *mess) {
9042 IFDBG(D_CONS, FN;);
9043 10943 dispatch_p_event(p2_slave_wait_vtbl, paxos, site, event, mess);
9044 10943 return 0;
9045 }
9046
9047 /* Phase 3 master wait */
9048 130508 static int paxos_fsm_p3_master_wait(pax_machine *paxos, site_def const *site,
9049 paxos_event event, pax_msg *mess) {
9050 IFDBG(D_CONS, FN;);
9051 130508 dispatch_p_event(p3_master_wait_vtbl, paxos, site, event, mess);
9052 130508 return 0;
9053 }
9054
9055 /* Phase 3 slave enter */
9056 115747 static int paxos_fsm_p3_slave_enter(pax_machine *paxos, site_def const *site,
9057 paxos_event event, pax_msg *mess) {
9058 (void)site;
9059 (void)event;
9060 (void)mess;
9061 IFDBG(D_CONS, FN;);
9062 /* Start timer */
9063 115747 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p3_slave_wait);
9064 115747 return 1;
9065 }
9066
9067 /* Phase 3 slave wait */
9068 115747 static int paxos_fsm_p3_slave_wait(pax_machine *paxos, site_def const *site,
9069 paxos_event event, pax_msg *mess) {
9070 IFDBG(D_CONS, FN;);
9071 115747 dispatch_p_event(p3_slave_wait_vtbl, paxos, site, event, mess);
9072 115747 return 0;
9073 }
9074
9075 /* Finished */
9076 172884 static int paxos_fsm_finished(pax_machine *paxos, site_def const *site,
9077 paxos_event event, pax_msg *mess) {
9078 IFDBG(D_CONS, FN;);
9079 172884 dispatch_p_event(p_finished_vtbl, paxos, site, event, mess);
9080 172884 return 0;
9081 }
9082
9083 #define X(b) #b
9084 const char *paxos_event_name[] = {p_events};
9085 #undef X
9086
9087 /* Trampoline which loops calling thunks pointed to by paxos->state.state_fp
9088 * until 0 is returned. */
9089 1303675 static void paxos_fsm(pax_machine *paxos, site_def const *site,
9090 paxos_event event, pax_msg *mess) {
9091 /* Crank the state machine until it stops */
9092 IFDBG(D_CONS, FN; PTREXP(paxos); SYCEXP(paxos->synode);
9093 BALCEXP(mess->proposal); STRLIT(paxos->state.state_name); STRLIT(" : ");
9094 STRLIT(paxos_event_name[event]));
9095
2/2
✓ Branch 0 taken 126690 times.
✓ Branch 1 taken 1303675 times.
 1430365 while (paxos->state.state_fp(paxos, site, event, mess)) {
9096 IFDBG(D_CONS, FN; PTREXP(paxos); SYCEXP(paxos->synode);
9097 BALCEXP(mess->proposal); STRLIT(paxos->state.state_name);
9098 STRLIT(" : "); STRLIT(paxos_event_name[event]));
9099 }
9100 1303675 }
9101